diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md index 3559905f..23e1a53e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Migrate to `TelemetryHandler` from `opentelemetry-util-genai` +- Remove `_StabilityMode` branching, `events.py`, and experimental test files + ## Version 2.2b0 (2025-12-19) - Fix overwritten log attributes in vertexai instrumentation ([#3925](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3925)) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py index 24d9d46a..a56cdcdf 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py @@ -47,57 +47,15 @@ wrap_function_wrapper, # type: ignore[reportUnknownVariableType] ) -from opentelemetry._logs import get_logger -from opentelemetry.instrumentation._semconv import ( - _OpenTelemetrySemanticConventionStability, - _OpenTelemetryStabilitySignalType, - _StabilityMode, -) from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from opentelemetry.instrumentation.utils import unwrap from opentelemetry.instrumentation.vertexai.package import _instruments -from opentelemetry.instrumentation.vertexai.patch import MethodWrappers +from opentelemetry.instrumentation.vertexai.patch import ( + agenerate_content, + generate_content, +) from opentelemetry.instrumentation.vertexai.utils import is_content_enabled -from opentelemetry.semconv.schemas import Schemas -from opentelemetry.trace import get_tracer -from opentelemetry.util.genai.completion_hook import load_completion_hook - - -def _methods_to_wrap( - method_wrappers: MethodWrappers, -): - # This import is very slow, do it lazily in case instrument() is not called - # pylint: disable=import-outside-toplevel - from google.cloud.aiplatform_v1.services.prediction_service import ( # noqa: PLC0415 - async_client, - client, - ) - from google.cloud.aiplatform_v1beta1.services.prediction_service import ( # noqa: PLC0415 - async_client as async_client_v1beta1, - ) - from google.cloud.aiplatform_v1beta1.services.prediction_service import ( # noqa: PLC0415 - client as client_v1beta1, - ) - - for client_class in ( - client.PredictionServiceClient, - client_v1beta1.PredictionServiceClient, - ): - yield ( - client_class, - client_class.generate_content.__name__, # type: ignore[reportUnknownMemberType] - method_wrappers.generate_content, - ) - - for client_class in ( - async_client.PredictionServiceAsyncClient, - async_client_v1beta1.PredictionServiceAsyncClient, - ): - yield ( - client_class, - client_class.generate_content.__name__, # type: ignore[reportUnknownMemberType] - method_wrappers.agenerate_content, - ) +from opentelemetry.util.genai.handler import get_telemetry_handler class VertexAIInstrumentor(BaseInstrumentor): @@ -110,61 +68,55 @@ def instrumentation_dependencies(self) -> Collection[str]: def _instrument(self, **kwargs: Any): """Enable VertexAI instrumentation.""" - completion_hook = ( - kwargs.get("completion_hook") or load_completion_hook() - ) - sem_conv_opt_in_mode = _OpenTelemetrySemanticConventionStability._get_opentelemetry_stability_opt_in_mode( - _OpenTelemetryStabilitySignalType.GEN_AI, - ) tracer_provider = kwargs.get("tracer_provider") - schema = ( - Schemas.V1_28_0.value - if sem_conv_opt_in_mode == _StabilityMode.DEFAULT - else Schemas.V1_36_0.value - ) - tracer = get_tracer( - __name__, - "", - tracer_provider, - schema_url=schema, - ) logger_provider = kwargs.get("logger_provider") - logger = get_logger( - __name__, - "", + meter_provider = kwargs.get("meter_provider") + + handler = get_telemetry_handler( + tracer_provider=tracer_provider, + meter_provider=meter_provider, logger_provider=logger_provider, - schema_url=schema, ) - sem_conv_opt_in_mode = _OpenTelemetrySemanticConventionStability._get_opentelemetry_stability_opt_in_mode( - _OpenTelemetryStabilitySignalType.GEN_AI, + + capture_content = is_content_enabled() + + # This import is very slow, do it lazily in case instrument() is not called + # pylint: disable=import-outside-toplevel + from google.cloud.aiplatform_v1.services.prediction_service import ( # noqa: PLC0415 + async_client, + client, ) - if sem_conv_opt_in_mode == _StabilityMode.DEFAULT: - # Type checker now knows sem_conv_opt_in_mode is a Literal[_StabilityMode.DEFAULT] - method_wrappers = MethodWrappers( - tracer, - logger, - is_content_enabled(sem_conv_opt_in_mode), - sem_conv_opt_in_mode, - completion_hook, - ) - elif sem_conv_opt_in_mode == _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL: - # Type checker now knows it's the other literal - method_wrappers = MethodWrappers( - tracer, - logger, - is_content_enabled(sem_conv_opt_in_mode), - sem_conv_opt_in_mode, - completion_hook, + from google.cloud.aiplatform_v1beta1.services.prediction_service import ( # noqa: PLC0415 + async_client as async_client_v1beta1, + ) + from google.cloud.aiplatform_v1beta1.services.prediction_service import ( # noqa: PLC0415 + client as client_v1beta1, + ) + + sync_wrapper = generate_content(capture_content, handler) + async_wrapper = agenerate_content(capture_content, handler) + + for client_class in ( + client.PredictionServiceClient, + client_v1beta1.PredictionServiceClient, + ): + method_name = client_class.generate_content.__name__ # type: ignore[reportUnknownMemberType] + wrap_function_wrapper( + client_class, + name=method_name, + wrapper=sync_wrapper, ) - else: - raise RuntimeError(f"{sem_conv_opt_in_mode} mode not supported") - for client_class, method_name, wrapper in _methods_to_wrap( - method_wrappers + self._methods_to_unwrap.append((client_class, method_name)) + + for client_class in ( + async_client.PredictionServiceAsyncClient, + async_client_v1beta1.PredictionServiceAsyncClient, ): + method_name = client_class.generate_content.__name__ # type: ignore[reportUnknownMemberType] wrap_function_wrapper( client_class, name=method_name, - wrapper=wrapper, + wrapper=async_wrapper, ) self._methods_to_unwrap.append((client_class, method_name)) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/events.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/events.py deleted file mode 100644 index 75969d16..00000000 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/events.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# type: ignore[reportUnknownDeprecated] - -""" -Factories for event types described in -https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-events.md#system-event. - -Hopefully this code can be autogenerated by Weaver once Gen AI semantic conventions are -schematized in YAML and the Weaver tool supports it. -""" - -from __future__ import annotations - -from dataclasses import asdict, dataclass -from typing import Any, Iterable, Literal - -from opentelemetry._logs import LogRecord -from opentelemetry.semconv._incubating.attributes import gen_ai_attributes -from opentelemetry.util.types import AnyValue - - -def user_event( - *, - role: str = "user", - content: AnyValue = None, -) -> LogRecord: - """Creates a User event - https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#user-event - """ - body: dict[str, AnyValue] = { - "role": role, - } - if content is not None: - body["content"] = content - return LogRecord( - event_name="gen_ai.user.message", - attributes={ - gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value, - }, - body=body, - ) - - -def assistant_event( - *, - role: str = "assistant", - content: AnyValue = None, -) -> LogRecord: - """Creates an Assistant event - https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#assistant-event - """ - body: dict[str, AnyValue] = { - "role": role, - } - if content is not None: - body["content"] = content - return LogRecord( - event_name="gen_ai.assistant.message", - attributes={ - gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value, - }, - body=body, - ) - - -def system_event( - *, - role: str = "system", - content: AnyValue = None, -) -> LogRecord: - """Creates a System event - https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#system-event - """ - body: dict[str, AnyValue] = { - "role": role, - } - if content is not None: - body["content"] = content - return LogRecord( - event_name="gen_ai.system.message", - attributes={ - gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value, - }, - body=body, - ) - - -def tool_event( - *, - role: str | None, - id_: str, - content: AnyValue = None, -) -> LogRecord: - """Creates a Tool message event - https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#event-gen_aitoolmessage - """ - if not role: - role = "tool" - - body: dict[str, AnyValue] = { - "role": role, - "id": id_, - } - if content is not None: - body["content"] = content - return LogRecord( - event_name="gen_ai.tool.message", - attributes={ - gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value, - }, - body=body, - ) - - -@dataclass -class ChoiceMessage: - """The message field for a gen_ai.choice event""" - - content: AnyValue = None - role: str = "assistant" - - -@dataclass -class ChoiceToolCall: - """The tool_calls field for a gen_ai.choice event""" - - @dataclass - class Function: - name: str - arguments: AnyValue = None - - function: Function - id: str - type: Literal["function"] = "function" - - -FinishReason = Literal[ - "content_filter", "error", "length", "stop", "tool_calls" -] - - -def choice_event( - *, - finish_reason: FinishReason | str, - index: int, - message: ChoiceMessage, - tool_calls: Iterable[ChoiceToolCall] = (), -) -> LogRecord: - """Creates a choice event, which describes the Gen AI response message. - https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#event-gen_aichoice - """ - body: dict[str, AnyValue] = { - "finish_reason": finish_reason, - "index": index, - "message": _asdict_filter_nulls(message), - } - - tool_calls_list = [ - _asdict_filter_nulls(tool_call) for tool_call in tool_calls - ] - if tool_calls_list: - body["tool_calls"] = tool_calls_list - - return LogRecord( - event_name="gen_ai.choice", - attributes={ - gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value, - }, - body=body, - ) - - -def _asdict_filter_nulls(instance: Any) -> dict[str, AnyValue]: - return asdict( - instance, - dict_factory=lambda kvs: {k: v for (k, v) in kvs if v is not None}, - ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index 481277d6..950da666 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -14,56 +14,43 @@ from __future__ import annotations -from contextlib import contextmanager -from dataclasses import asdict +import json from typing import ( TYPE_CHECKING, Any, - Awaitable, - Callable, - Literal, MutableSequence, - Union, - cast, - overload, ) -from opentelemetry._logs import Logger, LogRecord -from opentelemetry.instrumentation._semconv import ( - _StabilityMode, -) from opentelemetry.instrumentation.vertexai.utils import ( GenerateContentParams, _map_finish_reason, convert_content_to_message_parts, get_genai_request_attributes, - get_genai_response_attributes, get_server_attributes, - get_span_name, - request_to_events, - response_to_events, ) from opentelemetry.semconv._incubating.attributes import ( - gen_ai_attributes as GenAI, + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.semconv.attributes import ( + server_attributes as ServerAttributes, +) +from opentelemetry.util.genai.handler import ( + Error as InvocationError, +) +from opentelemetry.util.genai.handler import ( + TelemetryHandler, ) -from opentelemetry.trace import SpanKind, Tracer -from opentelemetry.util.genai.completion_hook import CompletionHook from opentelemetry.util.genai.types import ( - ContentCapturingMode, InputMessage, + LLMInvocation, OutputMessage, ) -from opentelemetry.util.genai.utils import gen_ai_json_dumps if TYPE_CHECKING: - from google.cloud.aiplatform_v1.services.prediction_service import client from google.cloud.aiplatform_v1.types import ( content, prediction_service, ) - from google.cloud.aiplatform_v1beta1.services.prediction_service import ( - client as client_v1beta1, - ) from google.cloud.aiplatform_v1beta1.types import ( content as content_v1beta1, ) @@ -109,263 +96,202 @@ def _extract_params( ) -# For details about GEN_AI_LATEST_EXPERIMENTAL stability mode see -# https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/gen-ai/gen-ai-agent-spans.md?plain=1#L18-L37 -class MethodWrappers: - @overload - def __init__( - self, - tracer: Tracer, - logger: Logger, - capture_content: ContentCapturingMode, - sem_conv_opt_in_mode: Literal[ - _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL - ], - completion_hook: CompletionHook, - ) -> None: ... - - @overload - def __init__( - self, - tracer: Tracer, - logger: Logger, - capture_content: bool, - sem_conv_opt_in_mode: Literal[_StabilityMode.DEFAULT], - completion_hook: CompletionHook, - ) -> None: ... - - def __init__( - self, - tracer: Tracer, - logger: Logger, - capture_content: Union[bool, ContentCapturingMode], - sem_conv_opt_in_mode: Union[ - Literal[_StabilityMode.DEFAULT], - Literal[_StabilityMode.GEN_AI_LATEST_EXPERIMENTAL], - ], - completion_hook: CompletionHook, - ) -> None: - self.tracer = tracer - self.logger = logger - self.capture_content = capture_content - self.sem_conv_opt_in_mode = sem_conv_opt_in_mode - self.completion_hook = completion_hook - - @contextmanager - def _with_new_instrumentation( - self, - capture_content: ContentCapturingMode, - instance: client.PredictionServiceClient - | client_v1beta1.PredictionServiceClient, - args: Any, - kwargs: Any, - ): - params = _extract_params(*args, **kwargs) - request_attributes = get_genai_request_attributes(True, params) - with self.tracer.start_as_current_span( - name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_attributes.get(GenAI.GEN_AI_REQUEST_MODEL, '')}".strip(), - kind=SpanKind.CLIENT, - ) as span: - - def handle_response( - response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse - | None, - ) -> None: - attributes = ( - get_server_attributes(instance.api_endpoint) # type: ignore[reportUnknownMemberType] - | request_attributes - | get_genai_response_attributes(response) - ) - event = LogRecord( - event_name="gen_ai.client.inference.operation.details", - ) - event.attributes = attributes.copy() - system_instructions, inputs, outputs = [], [], [] - if params.system_instruction: - system_instructions = convert_content_to_message_parts( +def _build_invocation( + params: GenerateContentParams, + api_endpoint: str, + capture_content: bool, +) -> LLMInvocation: + """Build an LLMInvocation from Vertex AI request parameters.""" + request_attributes = get_genai_request_attributes(params) + server_attrs = get_server_attributes(api_endpoint) + + # Build input messages + input_messages: list[InputMessage] = [] + if capture_content: + # Vertex AI uses a dedicated system_instruction field (equivalent to OpenAI's + # role="system") but its Content proto only supports role="user"|"model". + # We set role="system" so the emitter routes it to gen_ai.system_instructions. + if params.system_instruction: + input_messages.append( + InputMessage( + role="system", + parts=convert_content_to_message_parts( params.system_instruction + ), + ) + ) + if params.contents: + for c in params.contents: + input_messages.append( + InputMessage( + role=c.role or "user", + parts=convert_content_to_message_parts(c), ) - if params.contents: - inputs = [ - InputMessage( - role=content.role, - parts=convert_content_to_message_parts(content), - ) - for content in params.contents - ] - if response: - outputs = [ - OutputMessage( - finish_reason=_map_finish_reason( - candidate.finish_reason - ), - role=candidate.content.role, - parts=convert_content_to_message_parts( - candidate.content - ), - ) - for candidate in response.candidates - ] - self.completion_hook.on_completion( - inputs=inputs, - outputs=outputs, - system_instruction=system_instructions, - span=span, - log_record=event, ) - content_attributes = { - k: [asdict(x) for x in v] - for k, v in [ - ( - GenAI.GEN_AI_SYSTEM_INSTRUCTIONS, - system_instructions, - ), - (GenAI.GEN_AI_INPUT_MESSAGES, inputs), - (GenAI.GEN_AI_OUTPUT_MESSAGES, outputs), - ] - if v - } - if span.is_recording(): - span.set_attributes(attributes) - if capture_content in ( - ContentCapturingMode.SPAN_AND_EVENT, - ContentCapturingMode.SPAN_ONLY, - ): - span.set_attributes( - { - k: gen_ai_json_dumps(v) - for k, v in content_attributes.items() - } - ) - if capture_content in ( - ContentCapturingMode.SPAN_AND_EVENT, - ContentCapturingMode.EVENT_ONLY, - ): - event.attributes |= content_attributes - self.logger.emit(event) - - yield handle_response - - @contextmanager - def _with_default_instrumentation( - self, - capture_content: bool, - instance: client.PredictionServiceClient - | client_v1beta1.PredictionServiceClient, - args: Any, - kwargs: Any, - ): + + invocation = LLMInvocation( + request_model=request_attributes.get( + GenAIAttributes.GEN_AI_REQUEST_MODEL, "" + ), + input_messages=input_messages, + provider="vertex_ai", + framework="google-cloud-aiplatform", + server_address=server_attrs.get(ServerAttributes.SERVER_ADDRESS), + server_port=server_attrs.get(ServerAttributes.SERVER_PORT), + request_temperature=request_attributes.get( + GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE + ), + request_top_p=request_attributes.get( + GenAIAttributes.GEN_AI_REQUEST_TOP_P + ), + request_max_tokens=request_attributes.get( + GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS + ), + request_presence_penalty=request_attributes.get( + GenAIAttributes.GEN_AI_REQUEST_PRESENCE_PENALTY + ), + request_frequency_penalty=request_attributes.get( + GenAIAttributes.GEN_AI_REQUEST_FREQUENCY_PENALTY + ), + request_stop_sequences=list( + request_attributes.get( + GenAIAttributes.GEN_AI_REQUEST_STOP_SEQUENCES, [] + ) + ), + request_seed=request_attributes.get( + GenAIAttributes.GEN_AI_REQUEST_SEED + ), + ) + + # Propagate extra attributes that don't map to LLMInvocation fields + if GenAIAttributes.GEN_AI_OUTPUT_TYPE in request_attributes: + invocation.attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = ( + request_attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] + ) + + if capture_content and params.tools: + from google.protobuf import json_format as _jf + + tool_defs = [_jf.MessageToDict(t._pb) for t in params.tools] # type: ignore[union-attr] + invocation.attributes[GenAIAttributes.GEN_AI_TOOL_DEFINITIONS] = ( + json.dumps(tool_defs) + ) + + return invocation + + +def _apply_response_to_invocation( + invocation: LLMInvocation, + response: prediction_service.GenerateContentResponse + | prediction_service_v1beta1.GenerateContentResponse, + capture_content: bool, +) -> None: + """Apply response data to an existing LLMInvocation.""" + if hasattr(response, "usage_metadata") and response.usage_metadata: + invocation.input_tokens = response.usage_metadata.prompt_token_count + invocation.output_tokens = ( + response.usage_metadata.candidates_token_count + ) + + model = getattr(response, "model_version", None) + if model: + invocation.response_model_name = model + + finish_reasons = [] + output_messages: list[OutputMessage] = [] + for candidate in response.candidates: + fr = _map_finish_reason(candidate.finish_reason) + finish_reasons.append(fr) + parts = [] + if capture_content: + parts = convert_content_to_message_parts(candidate.content) + output_messages.append( + OutputMessage( + role=candidate.content.role or "model", + parts=parts, + finish_reason=fr, + ) + ) + + invocation.response_finish_reasons = finish_reasons + invocation.output_messages = output_messages + + +def generate_content(capture_content: bool, handler: TelemetryHandler): + """Wrap the sync `generate_content` method to trace it.""" + + def traced_method(wrapped, instance, args, kwargs): params = _extract_params(*args, **kwargs) api_endpoint: str = instance.api_endpoint # type: ignore[reportUnknownMemberType] - span_attributes = { - **get_genai_request_attributes(False, params), - **get_server_attributes(api_endpoint), - } - - span_name = get_span_name(span_attributes) - - with self.tracer.start_as_current_span( - name=span_name, - kind=SpanKind.CLIENT, - attributes=span_attributes, - ) as span: - for event in request_to_events( - params=params, capture_content=capture_content - ): - self.logger.emit(event) - - # TODO: set error.type attribute - # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md - - def handle_response( - response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse, - ) -> None: - if span.is_recording(): - # When streaming, this is called multiple times so attributes would be - # overwritten. In practice, it looks the API only returns the interesting - # attributes on the last streamed response. However, I couldn't find - # documentation for this and setting attributes shouldn't be too expensive. - span.set_attributes( - get_genai_response_attributes(response) - ) + invocation = _build_invocation(params, api_endpoint, capture_content) + handler.start_llm(invocation) + + try: + response = wrapped(*args, **kwargs) + except Exception as error: + try: # pragma: no cover - defensive + handler.fail_llm( + invocation, + InvocationError(message=str(error), type=type(error)), + ) + except Exception: + pass + raise + + try: + _apply_response_to_invocation( + invocation, response, capture_content + ) + handler.stop_llm(invocation) + except Exception as error: # pragma: no cover - defensive + try: + handler.fail_llm( + invocation, + InvocationError(message=str(error), type=type(error)), + ) + except Exception: + pass + + return response + + return traced_method + + +def agenerate_content(capture_content: bool, handler: TelemetryHandler): + """Wrap the async `generate_content` method to trace it.""" + + async def traced_method(wrapped, instance, args, kwargs): + params = _extract_params(*args, **kwargs) + api_endpoint: str = instance.api_endpoint # type: ignore[reportUnknownMemberType] + invocation = _build_invocation(params, api_endpoint, capture_content) + handler.start_llm(invocation) + + try: + response = await wrapped(*args, **kwargs) + except Exception as error: + try: # pragma: no cover - defensive + handler.fail_llm( + invocation, + InvocationError(message=str(error), type=type(error)), + ) + except Exception: + pass + raise + + try: + _apply_response_to_invocation( + invocation, response, capture_content + ) + handler.stop_llm(invocation) + except Exception as error: # pragma: no cover - defensive + try: + handler.fail_llm( + invocation, + InvocationError(message=str(error), type=type(error)), + ) + except Exception: + pass + + return response - for event in response_to_events( - response=response, capture_content=capture_content - ): - self.logger.emit(event) - - yield handle_response - - def generate_content( - self, - wrapped: Callable[ - ..., - prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse, - ], - instance: client.PredictionServiceClient - | client_v1beta1.PredictionServiceClient, - args: Any, - kwargs: Any, - ) -> ( - prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse - ): - if self.sem_conv_opt_in_mode == _StabilityMode.DEFAULT: - capture_content_bool = cast(bool, self.capture_content) - with self._with_default_instrumentation( - capture_content_bool, instance, args, kwargs - ) as handle_response: - response = wrapped(*args, **kwargs) - handle_response(response) - return response - else: - capture_content = cast(ContentCapturingMode, self.capture_content) - with self._with_new_instrumentation( - capture_content, instance, args, kwargs - ) as handle_response: - response = None - try: - response = wrapped(*args, **kwargs) - return response - finally: - handle_response(response) - - async def agenerate_content( - self, - wrapped: Callable[ - ..., - Awaitable[ - prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse - ], - ], - instance: client.PredictionServiceClient - | client_v1beta1.PredictionServiceClient, - args: Any, - kwargs: Any, - ) -> ( - prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse - ): - if self.sem_conv_opt_in_mode == _StabilityMode.DEFAULT: - capture_content_bool = cast(bool, self.capture_content) - with self._with_default_instrumentation( - capture_content_bool, instance, args, kwargs - ) as handle_response: - response = await wrapped(*args, **kwargs) - handle_response(response) - return response - else: - capture_content = cast(ContentCapturingMode, self.capture_content) - with self._with_new_instrumentation( - capture_content, instance, args, kwargs - ) as handle_response: - response = None - try: - response = await wrapped(*args, **kwargs) - return response - finally: - handle_response(response) + return traced_method diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py index 9686d0dd..0e232bfe 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py @@ -19,71 +19,42 @@ import logging import re from dataclasses import dataclass -from os import environ from typing import ( TYPE_CHECKING, - Iterable, - Literal, Mapping, Sequence, - Union, - cast, - overload, ) from urllib.parse import urlparse from google.protobuf import json_format -from opentelemetry._logs import LogRecord -from opentelemetry.instrumentation._semconv import ( - _StabilityMode, -) -from opentelemetry.instrumentation.vertexai.events import ( - ChoiceMessage, - ChoiceToolCall, - assistant_event, - choice_event, - system_event, - tool_event, - user_event, -) from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) from opentelemetry.semconv.attributes import server_attributes from opentelemetry.util.genai.types import ( - Blob, ContentCapturingMode, FinishReason, MessagePart, Text, - ToolCallRequest, ToolCallResponse, - Uri, ) from opentelemetry.util.genai.utils import get_content_capturing_mode -from opentelemetry.util.types import AnyValue, AttributeValue +from opentelemetry.util.types import AttributeValue if TYPE_CHECKING: from google.cloud.aiplatform_v1.types import ( content, - prediction_service, tool, ) from google.cloud.aiplatform_v1beta1.types import ( content as content_v1beta1, ) - from google.cloud.aiplatform_v1beta1.types import ( - prediction_service as prediction_service_v1beta1, - ) from google.cloud.aiplatform_v1beta1.types import ( tool as tool_v1beta1, ) -_MODEL = "model" - - @dataclass(frozen=True) class GenerateContentParams: model: str @@ -123,20 +94,15 @@ def get_server_attributes( def get_genai_request_attributes( # pylint: disable=too-many-branches - use_latest_semconvs: bool, params: GenerateContentParams, operation_name: GenAIAttributes.GenAiOperationNameValues = GenAIAttributes.GenAiOperationNameValues.CHAT, -): +) -> dict[str, AttributeValue]: model = _get_model_name(params.model) generation_config = params.generation_config attributes: dict[str, AttributeValue] = { GenAIAttributes.GEN_AI_OPERATION_NAME: operation_name.value, GenAIAttributes.GEN_AI_REQUEST_MODEL: model, } - if not use_latest_semconvs: - attributes[GenAIAttributes.GEN_AI_SYSTEM] = ( - GenAIAttributes.GenAiSystemValues.VERTEX_AI.value - ) if not generation_config: return attributes @@ -190,25 +156,6 @@ def get_genai_request_attributes( # pylint: disable=too-many-branches return attributes -def get_genai_response_attributes( - response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse - | None, -) -> dict[str, AttributeValue]: - if not response: - return {} - finish_reasons: list[str] = [ - _map_finish_reason(candidate.finish_reason) - for candidate in response.candidates - ] - return { - GenAIAttributes.GEN_AI_RESPONSE_MODEL: response.model_version, - GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS: finish_reasons, - GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS: response.usage_metadata.prompt_token_count, - GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS: response.usage_metadata.candidates_token_count, - } - - _MODEL_STRIP_RE = re.compile( r"^projects/(.*)/locations/(.*)/publishers/google/models/" ) @@ -218,113 +165,21 @@ def _get_model_name(model: str) -> str: return _MODEL_STRIP_RE.sub("", model) -OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = ( - "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" -) - - -@overload -def is_content_enabled( - mode: Literal[_StabilityMode.GEN_AI_LATEST_EXPERIMENTAL], -) -> ContentCapturingMode: ... - - -@overload -def is_content_enabled(mode: Literal[_StabilityMode.DEFAULT]) -> bool: ... - - -def is_content_enabled( - mode: Union[ - Literal[_StabilityMode.DEFAULT], - Literal[_StabilityMode.GEN_AI_LATEST_EXPERIMENTAL], - ], -) -> Union[bool, ContentCapturingMode]: - if mode == _StabilityMode.DEFAULT: - capture_content = environ.get( - OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, "false" - ) - - return capture_content.lower() == "true" - return get_content_capturing_mode() - - -def get_span_name(span_attributes: Mapping[str, AttributeValue]) -> str: - name = span_attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] - model = span_attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] - if not model: - return f"{name}" - return f"{name} {model}" - - -def request_to_events( - *, params: GenerateContentParams, capture_content: bool -) -> Iterable[LogRecord]: - # System message - if params.system_instruction: - request_content = _parts_to_any_value( - capture_content=capture_content, - parts=params.system_instruction.parts, - ) - yield system_event( - role=params.system_instruction.role, content=request_content - ) - - for content in params.contents or []: - # Assistant message - if content.role == _MODEL: - request_content = _parts_to_any_value( - capture_content=capture_content, parts=content.parts - ) - - yield assistant_event(role=content.role, content=request_content) - continue - - # Tool event - # - # Function call results can be parts inside of a user Content or in a separate Content - # entry without a role. That may cause duplication in a user event, see - # https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3280 - function_responses = [ - part.function_response - for part in content.parts - if "function_response" in part - ] - for idx, function_response in enumerate(function_responses): - yield tool_event( - id_=f"{function_response.name}_{idx}", - role=content.role, - content=json_format.MessageToDict( - function_response._pb.response # type: ignore[reportUnknownMemberType] - ) - if capture_content - else None, - ) - - if len(function_responses) == len(content.parts): - # If the content only contained function responses, don't emit a user event - continue - - request_content = _parts_to_any_value( - capture_content=capture_content, parts=content.parts - ) - yield user_event(role=content.role, content=request_content) - - -def _modality_from_mime_type(mime_type: str) -> str: - """Infer modality from MIME type prefix.""" - if mime_type.startswith("image/"): - return "image" - if mime_type.startswith("video/"): - return "video" - if mime_type.startswith("audio/"): - return "audio" - return mime_type +def is_content_enabled() -> bool: + """Check if content capturing is enabled via environment variable.""" + return get_content_capturing_mode() != ContentCapturingMode.NO_CONTENT def convert_content_to_message_parts( content: content.Content | content_v1beta1.Content, ) -> list[MessagePart]: - parts: MessagePart = [] + """Convert Vertex AI Content proto to a list of util-genai MessagePart objects. + + Only Text and ToolCallResponse parts are supported in this version. + Unsupported part types (inline_data, file_data, function_call) are + skipped until the corresponding util-genai types are available (HYBIM-604). + """ + parts: list[MessagePart] = [] for idx, part in enumerate(content.parts): if "function_response" in part: part = part.function_response @@ -335,118 +190,30 @@ def convert_content_to_message_parts( ) ) elif "function_call" in part: - part = part.function_call - parts.append( - ToolCallRequest( - id=f"{part.name}_{idx}", - name=part.name, - arguments=json_format.MessageToDict( - part._pb.args, # type: ignore[reportUnknownMemberType] - ), - ) + # ToolCallRequest not yet in util-genai (HYBIM-604) — skip + logging.debug( + "function_call part skipped (ToolCallRequest not yet supported)" ) elif "text" in part: parts.append(Text(content=part.text)) elif "inline_data" in part: - part = part.inline_data - mime_type = part.mime_type or "" - parts.append( - Blob( - mime_type=mime_type, - modality=_modality_from_mime_type(mime_type), - content=part.data or b"", - ) - ) + # Blob not yet in util-genai (HYBIM-604) — skip + logging.debug("inline_data part skipped (Blob not yet supported)") elif "file_data" in part: - part = part.file_data - mime_type = part.mime_type or "" - parts.append( - Uri( - mime_type=mime_type, - modality=_modality_from_mime_type(mime_type), - uri=part.file_uri or "", - ) - ) + # Uri not yet in util-genai (HYBIM-604) — skip + logging.debug("file_data part skipped (Uri not yet supported)") else: logging.warning("Unknown part dropped from telemetry %s", part) return parts -def response_to_events( - *, - response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse, - capture_content: bool, -) -> Iterable[LogRecord]: - for candidate in response.candidates: - tool_calls = _extract_tool_calls( - candidate=candidate, capture_content=capture_content - ) - - # The original function_call Part is still duplicated in message, see - # https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3280 - yield choice_event( - finish_reason=_map_finish_reason(candidate.finish_reason), - index=candidate.index, - # default to "model" since Vertex uses that instead of assistant - message=ChoiceMessage( - role=candidate.content.role or _MODEL, - content=_parts_to_any_value( - capture_content=capture_content, - parts=candidate.content.parts, - ), - ), - tool_calls=tool_calls, - ) - - -def _extract_tool_calls( - *, - candidate: content.Candidate | content_v1beta1.Candidate, - capture_content: bool, -) -> Iterable[ChoiceToolCall]: - for idx, part in enumerate(candidate.content.parts): - if "function_call" not in part: - continue - - yield ChoiceToolCall( - # Make up an id with index since vertex expects the indices to line up instead of - # using ids. - id=f"{part.function_call.name}_{idx}", - function=ChoiceToolCall.Function( - name=part.function_call.name, - arguments=json_format.MessageToDict( - part.function_call._pb.args # type: ignore[reportUnknownMemberType] - ) - if capture_content - else None, - ), - ) - - -def _parts_to_any_value( - *, - capture_content: bool, - parts: Sequence[content.Part] | Sequence[content_v1beta1.Part], -) -> list[dict[str, AnyValue]] | None: - if not capture_content: - return None - - return [ - cast( - "dict[str, AnyValue]", - type(part).to_dict( # type: ignore[reportUnknownMemberType] - part, always_print_fields_with_no_presence=False - ), - ) - for part in parts - ] - - def _map_finish_reason( finish_reason: content.Candidate.FinishReason - | content_v1beta1.Candidate.FinishReason, + | content_v1beta1.Candidate.FinishReason + | None, ) -> FinishReason | str: + if finish_reason is None: + return "error" EnumType = type(finish_reason) # pylint: disable=invalid-name if ( finish_reason is EnumType.FINISH_REASON_UNSPECIFIED diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_role.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_role.yaml deleted file mode 100644 index ed7590ba..00000000 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_role.yaml +++ /dev/null @@ -1,56 +0,0 @@ -interactions: -- request: - body: |- - { - "contents": [ - { - "role": "invalid_role", - "parts": [ - { - "text": "Say this is a test" - } - ] - } - ] - } - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '149' - Content-Type: - - application/json - User-Agent: - - python-requests/2.32.3 - method: POST - uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint - response: - body: - string: |- - { - "error": { - "code": 400, - "message": "Please use a valid role: user, model.", - "status": "INVALID_ARGUMENT", - "details": [] - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - Transfer-Encoding: - - chunked - Vary: - - Origin - - X-Origin - - Referer - content-length: - - '817' - status: - code: 400 - message: Bad Request -version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_without_events.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_no_content.yaml similarity index 100% rename from instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_without_events.yaml rename to instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_no_content.yaml diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml deleted file mode 100644 index 613f27ad..00000000 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml +++ /dev/null @@ -1,102 +0,0 @@ -interactions: -- request: - body: |- - { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "Say this is a test" - }, - { - "fileData": { - "mimeType": "image/jpeg", - "fileUri": "https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg" - } - }, - { - "inlineData": { - "mimeType": "image/jpeg", - "data": "iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==" - } - } - ] - } - ] - } - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '554' - Content-Type: - - application/json - User-Agent: - - python-requests/2.32.3 - method: POST - uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint - response: - body: - string: |- - { - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": "This is a test." - } - ] - }, - "finishReason": 1, - "avgLogprobs": -24.462081909179688 - } - ], - "usageMetadata": { - "promptTokenCount": 521, - "candidatesTokenCount": 5, - "totalTokenCount": 950, - "trafficType": 1, - "promptTokensDetails": [ - { - "modality": 2, - "tokenCount": 516 - }, - { - "modality": 1, - "tokenCount": 5 - } - ], - "candidatesTokensDetails": [ - { - "modality": 1, - "tokenCount": 5 - } - ], - "thoughtsTokenCount": 424 - }, - "modelVersion": "gemini-2.5-pro", - "createTime": "2025-10-13T16:29:47.639271Z", - "responseId": "-yjtaKeCJ5KYmecP76S4-AI" - } - headers: - Content-Type: - - application/json; charset=UTF-8 - Transfer-Encoding: - - chunked - Vary: - - Origin - - X-Origin - - Referer - content-length: - - '808' - status: - code: 200 - message: OK -version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml deleted file mode 100644 index 20a8db07..00000000 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml +++ /dev/null @@ -1,151 +0,0 @@ -interactions: -- request: - body: |- - { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "Get weather details in New Delhi and San Francisco?" - } - ] - }, - { - "role": "model", - "parts": [ - { - "functionCall": { - "name": "get_current_weather", - "args": { - "location": "New Delhi" - } - } - }, - { - "functionCall": { - "name": "get_current_weather", - "args": { - "location": "San Francisco" - } - } - } - ] - }, - { - "role": "user", - "parts": [ - { - "functionResponse": { - "name": "get_current_weather", - "response": { - "content": "{\"temperature\": 35, \"unit\": \"C\"}" - } - } - }, - { - "functionResponse": { - "name": "get_current_weather", - "response": { - "content": "{\"temperature\": 25, \"unit\": \"C\"}" - } - } - } - ] - } - ], - "tools": [ - { - "functionDeclarations": [ - { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": 6, - "properties": { - "location": { - "type": 1, - "description": "The location for which to get the weather. It can be a city name, a city name and state, or a zip code. Examples: 'San Francisco', 'San Francisco, CA', '95616', etc." - } - }, - "propertyOrdering": [ - "location" - ] - } - } - ] - } - ] - } - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '1731' - Content-Type: - - application/json - User-Agent: - - python-requests/2.32.3 - method: POST - uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint - response: - body: - string: |- - { - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": "The weather in New Delhi is 35\u00b0C and in San Francisco is 25\u00b0C.", - "thoughtSignature": "CqcOAcu98PDUiUq32HLxu6y5JNxvlcEjIaedPcBi5V86Hbf3vRgAXC4k0aMma0v1gotZVHrinF9edI9bEAdQdFR+2xOsaV1ntNeO4o35ymNNpm1rEv2p047eWxSABiXJ3VANecxNqQuVgxZyhClOn2BNmR/xGb43REabpcMzboVGVT6iKSJ/g3sCIt4bddY1IQ5zTdSV7lvCyZLuu9736VCJjULzAslhxSb9/xBlQu/pvcak6CmFFLZuDamqeJ3RvhalpklF6qz/Eq9nhhpdRPTERmGU7mNe4fiSpql6JxelO56ksKKGzSG+USa9vxcCQRRgAoaLlMHRegwPjfoA4hHgbpLBIUk9BNXrzmW5APOpJO3rqgIvKQ/WsZzIBI3l+slI51WHgVLr4qNvDUe8VKLhbApVj3L+rb5dQ/u6U9V2oSVA5pQFt6Lqubyjg8yCl0q+0dsvIYUqJSaDRONMfS5hwVu+RTU/SOEXLwTNEiDDuj3wO8xQuyitDAHqya797sOH2ThlSoT/c7V3Du00TBCk0eq8XaxyTobPMCaKNbrqjRTfMn5fortZlPwyAJqoTZsaSeaRufCFpgBMy34/QcLiCdVwKN6rLgBpDhtzThnyBzavDP2ltu8sxWEisJXEOel0q8LvBqTqlFY9dWrRScnB/TvitGnwqlV0PVpWU60wH/2Agexw0+aFlMRYxiFRjuHPK9R2UfLbEzVpDC8M+f16QRZOnWQc8L+cxEW/xHT6hCUxcTmSrpldAs2Ss8A21NcPTlr0lNviZxB+4xDq1dxB/VMS/qOnGVbr11nEOOVj+bWJyP/LLTjdUigGs+ISAzmEr2zkUirLdlmz/Mz1YXk9CMIus8WlPVzhQZuXY8dhSKxks0w8bJ5TjJelAwPGFgz6iWSz44IeaH22Fxdvub28/mdmvy3OerI3vfwlePUC9L8n2cg7gZjIE59PXi2x1EUTJfvcBrVTO5qqPuRpJZ9/d3ryEtS9tHsMnzfcEiS2lxNb+EqKaHk74hqvt6vEA+eQaYtMqG5TO79dpGDt0UD1cJEwngfMiL85+5qWHW4g2hSSJYgg55GtdZbxdGs960p8G02FbqUOSVcDGcIBDCkvFSGHJqMATvYQzChiwP2AnDVLN0aWd43UEl3rHESdOGLOqrxetcWmVsDacowhiy6jleS/Xml7d8mqqs2483XCOVMMYPcFh9budRed7PIf7eAYhWzGwt123JLbxqwBOBf1MHUJl2OQBy62HCJLYMlRhr1s2KVRK9Een5cgUa3H07r/s46YS2A0KOP/B0Ub0MflvGBAgo8LABj5cY2Ao1hTKcnBjgJPTNS0awETbtzU3Y74SPMsJ1Ipu2o76TFS3YFq1Fbit1VsceuYxzcqU04NFBtXAU7YojLv9LBOZXk6SukOWI/2cNUOHlzlWsSi34heVFGH4AcDbhc0/QZHhjSTE9FkBY9feTtSnTR16Q1bg3K9+7rxArD51Wj8q1ZNhVtZrzpuWGqoneI0WqJ25U5f1jSzQFW2HyzmIs/B5KiFEk4Do0EiaVcCQ5/1UGReYbgtSKa8PAvUGh2Lqiev1VMl6i4cDs7s3U9swvZdBdK+SauYC7UqEVov1G8UR8Bth2YncqUPSv7W2X7ppA6PwDXSCqCSgp3wnfEW8tQIa6AQW2s4Udp0lbYWr3Vu2DSK3iu8mVa3ybvHs5Go4SR9zJ8xElMn4adprz8jokYpoXK94DVeaUetKqTw6X3FzRb+K+e/W0MHZVEQtAjk2wwcNqAfs8JdXi2Mh/l+0PFp6Hhz0rI1rzZ3B08lVR+QU+0XpN5rerUmq44oE6t+dP6Cv8yl5kVBCjvtHZ9DLUGeSEofOR3JXs34rkWeOLfQ+vRZkYoiIRKkqFhf/RnRoU18vmfsbvK9C3XcO3wmfo58Iw2uYd9L/p6VdB27pPUiBJbTAzXbEA3CWUbedMg2vDZiucS4iBkWlmN91CofwgmNfiImdcauz5ug3TC5qRWUIqX/nNMSscPjpbhp3nmja727fyH1p4ytpXQbRX0do28KpY5XQ9yF9gAWxG2zHxtBGiZVG4iXRlpZlfmFUc4kQLUTrInYUoagh9zF6Cxwrz7smRqgOkWKOYiOTiR9ofTkvpmQpdGpDOsxjcPRd3V+jjkfZwT5yzsM7IOAzzhZKKvPWgfJfzaAI43W+dUWiMWzM5KX7Wliu5J3uEXtldo0lW2WpOiuFK5GIPBh5482IbSAtA14YEplOYVCrCI9UG356TImFYyCLuNKAxbzA3U9TSJCk0tUbssWKq8aIaoEbtvE43P1G8x5teVFR5Wu1nZypgKgYFSS5tRL+niDV82yaCOTIGjdA6yFHq8gRlv9xkmZjyPIB4LfAhAIEz9X0+hNvZBfTofi1UXRh/54GIqr1YIXmJVYKb/srAU5Qne5lQdvFauFIQbnlESVUeb1RA==" - } - ] - }, - "finishReason": 1, - "avgLogprobs": -1.18875130740079 - } - ], - "usageMetadata": { - "promptTokenCount": 128, - "candidatesTokenCount": 22, - "totalTokenCount": 575, - "trafficType": 1, - "promptTokensDetails": [ - { - "modality": 1, - "tokenCount": 128 - } - ], - "candidatesTokensDetails": [ - { - "modality": 1, - "tokenCount": 22 - } - ], - "thoughtsTokenCount": 425 - }, - "modelVersion": "gemini-2.5-pro", - "createTime": "2025-08-19T14:55:39.882212Z", - "responseId": "a5CkaKTsNa3hgLUP59n1oQo" - } - headers: - Content-Type: - - application/json; charset=UTF-8 - Transfer-Encoding: - - chunked - Vary: - - Origin - - X-Origin - - Referer - content-length: - - '3277' - status: - code: 200 - message: OK -version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py index 0cf7ba0c..5a5619ef 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py @@ -30,15 +30,13 @@ GenerativeModel, ) -from opentelemetry.instrumentation._semconv import ( - OTEL_SEMCONV_STABILITY_OPT_IN, - _OpenTelemetrySemanticConventionStability, -) from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor -from opentelemetry.instrumentation.vertexai.utils import ( +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.util.genai import handler as genai_handler +from opentelemetry.util.genai.environment_variables import ( OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, + OTEL_INSTRUMENTATION_GENAI_EMITTERS, ) -from opentelemetry.sdk._logs import LoggerProvider # Backward compatibility for InMemoryLogExporter -> InMemoryLogRecordExporter rename try: @@ -109,6 +107,26 @@ def fixture_meter_provider(metric_reader): ) +@pytest.fixture(autouse=True) +def environment(): + """Reset TelemetryHandler singleton and evaluator config between tests.""" + original_evals = os.environ.get( + "OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS" + ) + os.environ["OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS"] = "none" + genai_handler.TelemetryHandler._reset_for_testing() + + yield + + if original_evals is None: + os.environ.pop("OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS", None) + else: + os.environ["OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS"] = ( + original_evals + ) + genai_handler.TelemetryHandler._reset_for_testing() + + @pytest.fixture(autouse=True) def vertexai_init(vcr: VCR) -> None: # When not recording (in CI), don't do any auth. That prevents trying to read application @@ -125,67 +143,16 @@ def vertexai_init(vcr: VCR) -> None: @pytest.fixture(scope="function") -def instrument_no_content( - tracer_provider, logger_provider, meter_provider, request -): - # Reset global state.. - _OpenTelemetrySemanticConventionStability._initialized = False - os.environ.update({OTEL_SEMCONV_STABILITY_OPT_IN: "stable"}) +def instrument_no_content(tracer_provider, logger_provider, meter_provider): os.environ.update( {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "False"} ) - - instrumentor = VertexAIInstrumentor() - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - - yield instrumentor - os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) - if instrumentor.is_instrumented_by_opentelemetry: - instrumentor.uninstrument() - - -@pytest.fixture(scope="function") -def instrument_no_content_with_experimental_semconvs( - tracer_provider, logger_provider, meter_provider, request -): - # Reset global state.. - _OpenTelemetrySemanticConventionStability._initialized = False - os.environ.update( - {OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental"} - ) os.environ.update( - {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "NO_CONTENT"} + {OTEL_INSTRUMENTATION_GENAI_EMITTERS: "span_metric_event"} ) - instrumentor = VertexAIInstrumentor() - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - - yield instrumentor - os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) - if instrumentor.is_instrumented_by_opentelemetry: - instrumentor.uninstrument() - + genai_handler.TelemetryHandler._reset_for_testing() -@pytest.fixture(scope="function") -def instrument_with_experimental_semconvs( - tracer_provider, logger_provider, meter_provider -): - # Reset global state.. - _OpenTelemetrySemanticConventionStability._initialized = False - os.environ.update( - {OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental"} - ) - os.environ.update( - {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "SPAN_AND_EVENT"} - ) instrumentor = VertexAIInstrumentor() instrumentor.instrument( tracer_provider=tracer_provider, @@ -195,49 +162,22 @@ def instrument_with_experimental_semconvs( yield instrumentor os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_EMITTERS, None) if instrumentor.is_instrumented_by_opentelemetry: instrumentor.uninstrument() @pytest.fixture(scope="function") -def instrument_with_upload_hook( - tracer_provider, logger_provider, meter_provider -): - # Reset global state.. - _OpenTelemetrySemanticConventionStability._initialized = False +def instrument_with_content(tracer_provider, logger_provider, meter_provider): os.environ.update( - { - OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental", - "OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK": "upload", - "OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH": "memory://", - OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "SPAN_AND_EVENT", - } + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "True"} ) - instrumentor = VertexAIInstrumentor() - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, + os.environ.update( + {OTEL_INSTRUMENTATION_GENAI_EMITTERS: "span_metric_event"} ) - yield instrumentor - os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) - os.environ.pop("OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK", None) - os.environ.pop("OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH", None) - if instrumentor.is_instrumented_by_opentelemetry: - instrumentor.uninstrument() + genai_handler.TelemetryHandler._reset_for_testing() - -@pytest.fixture(scope="function") -def instrument_with_content( - tracer_provider, logger_provider, meter_provider, request -): - # Reset global state.. - _OpenTelemetrySemanticConventionStability._initialized = False - os.environ.update({OTEL_SEMCONV_STABILITY_OPT_IN: "stable"}) - os.environ.update( - {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "True"} - ) instrumentor = VertexAIInstrumentor() instrumentor.instrument( tracer_provider=tracer_provider, @@ -247,6 +187,7 @@ def instrument_with_content( yield instrumentor os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_EMITTERS, None) if instrumentor.is_instrumented_by_opentelemetry: instrumentor.uninstrument() diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py index 05e729e1..eb0a6892 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json + import pytest from google.api_core.exceptions import BadRequest, NotFound from vertexai.generative_models import ( @@ -31,6 +33,21 @@ from opentelemetry.trace import StatusCode +def assert_handler_event(log, parent_span): + """Assert log record is the unified GenAI handler event and return its body.""" + assert ( + log.log_record.event_name + == "gen_ai.client.inference.operation.details" + ) + assert log.log_record.body is not None + if parent_span: + span_context = parent_span.get_span_context() + assert log.log_record.trace_id == span_context.trace_id + assert log.log_record.span_id == span_context.span_id + assert log.log_record.trace_flags == span_context.trace_flags + return dict(log.log_record.body) + + @pytest.mark.vcr() def test_generate_content( span_exporter: InMemorySpanExporter, @@ -54,53 +71,62 @@ def test_generate_content( # Emits span spans = span_exporter.get_finished_spans() assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", - "gen_ai.usage.input_tokens": 5, - "gen_ai.usage.output_tokens": 5, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } + span = spans[0] + assert span.name == "chat gemini-2.5-pro" + + # Core span attributes + attrs = dict(span.attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.response.finish_reasons"] == ("stop",) + assert attrs["gen_ai.response.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert attrs["gen_ai.usage.input_tokens"] == 5 + assert attrs["gen_ai.usage.output_tokens"] == 5 + assert attrs["server.address"] == "us-central1-aiplatform.googleapis.com" + assert attrs["server.port"] == 443 + + # Content on span (JSON-encoded input/output messages) + assert "gen_ai.input.messages" in attrs + input_msgs = json.loads(attrs["gen_ai.input.messages"]) + assert input_msgs == [ + { + "role": "user", + "parts": [{"type": "text", "content": "Say this is a test"}], + } + ] - logs = log_exporter.get_finished_logs() + assert "gen_ai.output.messages" in attrs + output_msgs = json.loads(attrs["gen_ai.output.messages"]) + assert output_msgs == [ + { + "role": "model", + "parts": [{"type": "text", "content": "This is a test."}], + "finish_reason": "stop", + } + ] - # Emits user and choice events - assert len(logs) == 2 - user_log, choice_log = [log_data.log_record for log_data in logs] - - span_context = spans[0].get_span_context() - assert user_log.trace_id == span_context.trace_id - assert user_log.span_id == span_context.span_id - assert user_log.trace_flags == span_context.trace_flags - assert user_log.attributes == {"gen_ai.system": "vertex_ai"} - assert user_log.event_name == "gen_ai.user.message" - assert user_log.body == { - "content": [{"text": "Say this is a test"}], - "role": "user", - } - - assert choice_log.trace_id == span_context.trace_id - assert choice_log.span_id == span_context.span_id - assert choice_log.trace_flags == span_context.trace_flags - assert choice_log.attributes == {"gen_ai.system": "vertex_ai"} - assert choice_log.event_name == "gen_ai.choice" - assert choice_log.body == { - "finish_reason": "stop", - "index": 0, - "message": { - "content": [{"text": "This is a test."}], + # Content events emitter emits a single event + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + body = assert_handler_event(logs[0], span) + assert body["gen_ai.input.messages"] == [ + { + "role": "user", + "parts": [{"type": "text", "content": "Say this is a test"}], + } + ] + assert body["gen_ai.output.messages"] == [ + { "role": "model", - }, - } + "parts": [{"type": "text", "content": "This is a test."}], + "finish_reason": "stop", + } + ] @pytest.mark.vcr() -def test_generate_content_without_events( +def test_generate_content_no_content( span_exporter: InMemorySpanExporter, log_exporter: InMemoryLogRecordExporter, generate_content: callable, @@ -114,37 +140,29 @@ def test_generate_content_without_events( ], ) - # Emits span + # Emits span without content attributes spans = span_exporter.get_finished_spans() assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", - "gen_ai.usage.input_tokens": 5, - "gen_ai.usage.output_tokens": 5, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } - + span = spans[0] + assert span.name == "chat gemini-2.5-pro" + attrs = dict(span.attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.response.finish_reasons"] == ("stop",) + assert attrs["gen_ai.response.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert attrs["gen_ai.usage.input_tokens"] == 5 + assert attrs["gen_ai.usage.output_tokens"] == 5 + assert attrs["server.address"] == "us-central1-aiplatform.googleapis.com" + assert attrs["server.port"] == 443 + + # No content attributes on span + assert "gen_ai.input.messages" not in attrs + assert "gen_ai.output.messages" not in attrs + + # No events emitted when content is disabled logs = log_exporter.get_finished_logs() - # Emits user and choice event without body.content - assert len(logs) == 2 - user_log, choice_log = [log_data.log_record for log_data in logs] - assert user_log.attributes == {"gen_ai.system": "vertex_ai"} - assert user_log.event_name == "gen_ai.user.message" - assert user_log.body == {"role": "user"} - - assert choice_log.attributes == {"gen_ai.system": "vertex_ai"} - assert choice_log.event_name == "gen_ai.choice" - assert choice_log.body == { - "finish_reason": "stop", - "index": 0, - "message": {"role": "model"}, - } + assert len(logs) == 0 @pytest.mark.vcr() @@ -168,15 +186,13 @@ def test_generate_content_empty_model( spans = span_exporter.get_finished_spans() assert len(spans) == 1 - assert spans[0].name == "chat" - # Captures invalid params - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "", - "gen_ai.system": "vertex_ai", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } + assert spans[0].name == "chat " + attrs = dict(spans[0].attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.model"] == "" + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert attrs["server.address"] == "us-central1-aiplatform.googleapis.com" + assert attrs["server.port"] == 443 assert_span_error(spans[0]) @@ -202,14 +218,12 @@ def test_generate_content_missing_model( spans = span_exporter.get_finished_spans() assert len(spans) == 1 assert spans[0].name == "chat gemini-does-not-exist" - # Captures invalid params - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-does-not-exist", - "gen_ai.system": "vertex_ai", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } + attrs = dict(spans[0].attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.model"] == "gemini-does-not-exist" + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert attrs["server.address"] == "us-central1-aiplatform.googleapis.com" + assert attrs["server.port"] == 443 assert_span_error(spans[0]) @@ -237,52 +251,16 @@ def test_generate_content_invalid_temperature( spans = span_exporter.get_finished_spans() assert len(spans) == 1 assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.request.temperature": 1000.0, - "gen_ai.system": "vertex_ai", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } + attrs = dict(spans[0].attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.request.temperature"] == 1000.0 + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert attrs["server.address"] == "us-central1-aiplatform.googleapis.com" + assert attrs["server.port"] == 443 assert_span_error(spans[0]) -@pytest.mark.vcr() -def test_generate_content_invalid_role( - log_exporter: InMemoryLogRecordExporter, - generate_content: callable, - instrument_with_content: VertexAIInstrumentor, -): - model = GenerativeModel("gemini-2.5-pro") - try: - # Fails because role must be "user" or "model" - generate_content( - model, - [ - Content( - role="invalid_role", - parts=[Part.from_text("Say this is a test")], - ) - ], - ) - except BadRequest: - pass - - # Emits the faulty content which caused the request to fail - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - log = logs[0].log_record - assert log.attributes == { - "gen_ai.system": "vertex_ai", - } - assert log.event_name == "gen_ai.user.message" - assert log.body == { - "content": [{"text": "Say this is a test"}], - "role": "invalid_role", - } - - @pytest.mark.vcr() def test_generate_content_extra_params( span_exporter, @@ -310,78 +288,78 @@ def test_generate_content_extra_params( spans = span_exporter.get_finished_spans() assert len(spans) == 1 - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.frequency_penalty": 1.0, - "gen_ai.request.max_tokens": 5, - "gen_ai.request.seed": 12345, - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.request.presence_penalty": -1.5, - "gen_ai.request.stop_sequences": ("\n\n\n",), - "gen_ai.request.temperature": 0.20000000298023224, - "gen_ai.request.top_p": 0.949999988079071, - "gen_ai.response.finish_reasons": ("length",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", - "gen_ai.usage.input_tokens": 5, - "gen_ai.usage.output_tokens": 0, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } - - -def assert_span_error(span: ReadableSpan) -> None: + attrs = dict(spans[0].attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.frequency_penalty"] == 1.0 + assert attrs["gen_ai.request.max_tokens"] == 5 + assert attrs["gen_ai.request.seed"] == 12345 + assert attrs["gen_ai.request.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.request.presence_penalty"] == -1.5 + assert attrs["gen_ai.request.stop_sequences"] == ("\n\n\n",) + assert attrs["gen_ai.request.temperature"] == 0.20000000298023224 + assert attrs["gen_ai.request.top_p"] == 0.949999988079071 + assert attrs["gen_ai.response.finish_reasons"] == ("length",) + assert attrs["gen_ai.response.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert attrs["gen_ai.usage.input_tokens"] == 5 + assert attrs["gen_ai.usage.output_tokens"] == 0 + assert attrs["server.address"] == "us-central1-aiplatform.googleapis.com" + assert attrs["server.port"] == 443 + + +def assert_span_error(span: ReadableSpan, error_type: str = None) -> None: # Sets error status assert span.status.status_code == StatusCode.ERROR - # TODO: check thate error.type is set - # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md - - # Records exception event - error_events = [e for e in span.events if e.name == "exception"] - assert error_events != [] + # TelemetryHandler sets error.type attribute + attrs = dict(span.attributes) + assert "error.type" in attrs + if error_type: + assert attrs["error.type"] == error_type @pytest.mark.vcr() def test_generate_content_all_events( + span_exporter: InMemorySpanExporter, log_exporter: InMemoryLogRecordExporter, generate_content: callable, instrument_with_content: VertexAIInstrumentor, ): - generate_content_all_input_events( + generate_content_all_input_messages( GenerativeModel( "gemini-2.5-pro", system_instruction=Part.from_text( "You are a clever language model" ), ), + span_exporter, log_exporter, - instrument_with_content, ) @pytest.mark.vcr() def test_preview_generate_content_all_input_events( + span_exporter: InMemorySpanExporter, log_exporter: InMemoryLogRecordExporter, generate_content: callable, instrument_with_content: VertexAIInstrumentor, ): - generate_content_all_input_events( + generate_content_all_input_messages( PreviewGenerativeModel( "gemini-2.5-pro", system_instruction=Part.from_text( "You are a clever language model" ), ), + span_exporter, log_exporter, - instrument_with_content, ) -def generate_content_all_input_events( +def generate_content_all_input_messages( model: GenerativeModel | PreviewGenerativeModel, + span_exporter: InMemorySpanExporter, log_exporter: InMemoryLogRecordExporter, - instrument_with_content: VertexAIInstrumentor, ): model.generate_content( [ @@ -403,49 +381,48 @@ def generate_content_all_input_events( ), ) - # Emits a system event, 2 users events, an assistant event, and the choice (response) event - logs = log_exporter.get_finished_logs() - assert len(logs) == 5 - system_log, user_log1, assistant_log, user_log2, choice_log = [ - log_data.log_record for log_data in logs + # Emits span with content attributes + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + attrs = dict(span.attributes) + + # Verify system instructions are on the span + assert "gen_ai.system_instructions" in attrs + + # Verify input messages on span (system messages excluded from input) + assert "gen_ai.input.messages" in attrs + input_msgs = json.loads(attrs["gen_ai.input.messages"]) + assert len(input_msgs) == 3 + assert input_msgs[0]["role"] == "user" + assert input_msgs[0]["parts"] == [ + {"type": "text", "content": "My name is OpenTelemetry"} + ] + assert input_msgs[1]["role"] == "model" + assert input_msgs[1]["parts"] == [ + {"type": "text", "content": "Hello OpenTelemetry!"} + ] + assert input_msgs[2]["role"] == "user" + assert input_msgs[2]["parts"] == [ + { + "type": "text", + "content": "Address me by name and say this is a test", + } ] - assert system_log.attributes == {"gen_ai.system": "vertex_ai"} - assert system_log.event_name == "gen_ai.system.message" - assert system_log.body == { - "content": [{"text": "You are a clever language model"}], - # The API only allows user and model, so system instruction is considered a user role - "role": "user", - } - - assert user_log1.attributes == {"gen_ai.system": "vertex_ai"} - assert user_log1.event_name == "gen_ai.user.message" - assert user_log1.body == { - "content": [{"text": "My name is OpenTelemetry"}], - "role": "user", - } - - assert assistant_log.attributes == {"gen_ai.system": "vertex_ai"} - assert assistant_log.event_name == "gen_ai.assistant.message" - assert assistant_log.body == { - "content": [{"text": "Hello OpenTelemetry!"}], - "role": "model", - } - - assert user_log2.attributes == {"gen_ai.system": "vertex_ai"} - assert user_log2.event_name == "gen_ai.user.message" - assert user_log2.body == { - "content": [{"text": "Address me by name and say this is a test"}], - "role": "user", - } - - assert choice_log.attributes == {"gen_ai.system": "vertex_ai"} - assert choice_log.event_name == "gen_ai.choice" - assert choice_log.body == { - "finish_reason": "stop", - "index": 0, - "message": { - "content": [{"text": "OpenTelemetry, this is a test."}], - "role": "model", - }, - } + # Verify output messages on span + assert "gen_ai.output.messages" in attrs + output_msgs = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_msgs) == 1 + assert output_msgs[0]["role"] == "model" + assert output_msgs[0]["parts"] == [ + {"type": "text", "content": "OpenTelemetry, this is a test."} + ] + assert output_msgs[0]["finish_reason"] == "stop" + + # Content events emitter emits a single event + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + body = assert_handler_event(logs[0], span) + assert "gen_ai.input.messages" in body + assert "gen_ai.output.messages" in body diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py deleted file mode 100644 index 617d9a81..00000000 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py +++ /dev/null @@ -1,498 +0,0 @@ -from __future__ import annotations - -import pytest -from google.api_core.exceptions import BadRequest, NotFound -from vertexai.generative_models import ( - Content, - GenerationConfig, - GenerativeModel, - Image, - Part, -) -from vertexai.preview.generative_models import ( - GenerativeModel as PreviewGenerativeModel, -) - -from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor - -# Backward compatibility for InMemoryLogExporter -> InMemoryLogRecordExporter rename -try: - from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import ( # pylint: disable=no-name-in-module - InMemoryLogRecordExporter, - ) -except ImportError: - # Fallback to old name for compatibility with older SDK versions - from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import ( - InMemoryLogExporter as InMemoryLogRecordExporter, - ) -from opentelemetry.sdk.trace import ReadableSpan -from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( - InMemorySpanExporter, -) -from opentelemetry.trace import StatusCode - - -@pytest.mark.vcr() -def test_generate_content_with_files( - span_exporter: InMemorySpanExporter, - log_exporter: InMemoryLogRecordExporter, - generate_content: callable, - instrument_with_experimental_semconvs: VertexAIInstrumentor, -): - model = GenerativeModel("gemini-2.5-pro") - generate_content( - model, - [ - Content( - role="user", - parts=[ - Part.from_text("Say this is a test"), - Part.from_uri( - mime_type="image/jpeg", - uri="https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg", - ), - Part.from_image( - Image.from_bytes( - "iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==" - ) - ), - ], - ), - ], - ) - - # Emits span - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.usage.input_tokens": 521, - "gen_ai.usage.output_tokens": 5, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"},{"mime_type":"image/jpeg","modality":"image","uri":"https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg","type":"uri"},{"mime_type":"image/jpeg","modality":"image","content":"iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==","type":"blob"}]}]', - "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]', - } - - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - log = logs[0].log_record - assert log.attributes == { - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.usage.input_tokens": 521, - "gen_ai.usage.output_tokens": 5, - "gen_ai.input.messages": ( - { - "role": "user", - "parts": ( - {"content": "Say this is a test", "type": "text"}, - { - "mime_type": "image/jpeg", - "modality": "image", - "uri": "https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg", - "type": "uri", - }, - { - "mime_type": "image/jpeg", - "modality": "image", - "content": b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x05\x00\x00\x00\x05\x08\x06\x00\x00\x00\x8do&\xe5\x00\x00\x00\x1cIDAT\x08\xd7c\xf8\xff\xff?\xc3\x7f\x06 \x05\xc3 \x12\x84\xd01\xf1\x82X\xcd\x04\x00\x0e\xf55\xcb\xd1\x8e\x0e\x1f\x00\x00\x00\x00IEND\xaeB`\x82", - "type": "blob", - }, - ), - }, - ), - "gen_ai.output.messages": ( - { - "role": "model", - "parts": ({"content": "This is a test.", "type": "text"},), - "finish_reason": "stop", - }, - ), - } - - -@pytest.mark.vcr() -def test_generate_content_without_events( - span_exporter: InMemorySpanExporter, - log_exporter: InMemoryLogRecordExporter, - generate_content: callable, - instrument_with_experimental_semconvs: VertexAIInstrumentor, -): - model = GenerativeModel("gemini-2.5-pro") - generate_content( - model, - [ - Content(role="user", parts=[Part.from_text("Say this is a test")]), - ], - ) - - # Emits span - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]', - "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.usage.input_tokens": 5, - "gen_ai.usage.output_tokens": 5, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } - - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - log = logs[0].log_record - print(log.attributes) - assert log.attributes == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.usage.input_tokens": 5, - "gen_ai.usage.output_tokens": 5, - "gen_ai.input.messages": ( - { - "role": "user", - "parts": ({"content": "Say this is a test", "type": "text"},), - }, - ), - "gen_ai.output.messages": ( - { - "role": "model", - "parts": ({"content": "This is a test.", "type": "text"},), - "finish_reason": "stop", - }, - ), - } - - -@pytest.mark.vcr() -def test_generate_content_empty_model( - span_exporter: InMemorySpanExporter, - generate_content: callable, - instrument_with_experimental_semconvs: VertexAIInstrumentor, -): - model = GenerativeModel("") - try: - generate_content( - model, - [ - Content( - role="user", parts=[Part.from_text("Say this is a test")] - ) - ], - ) - except ValueError: - pass - - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "chat" - # Captures invalid params - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', - } - assert_span_error(spans[0]) - - -@pytest.mark.vcr() -def test_generate_content_missing_model( - span_exporter: InMemorySpanExporter, - generate_content: callable, - instrument_with_experimental_semconvs: VertexAIInstrumentor, -): - model = GenerativeModel("gemini-does-not-exist") - try: - generate_content( - model, - [ - Content( - role="user", parts=[Part.from_text("Say this is a test")] - ) - ], - ) - except NotFound: - pass - - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "chat gemini-does-not-exist" - # Captures invalid params - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-does-not-exist", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', - } - assert_span_error(spans[0]) - - -@pytest.mark.vcr() -def test_generate_content_invalid_temperature( - span_exporter: InMemorySpanExporter, - generate_content: callable, - instrument_with_experimental_semconvs: VertexAIInstrumentor, -): - model = GenerativeModel("gemini-2.5-pro") - try: - # Temperature out of range causes error - generate_content( - model, - [ - Content( - role="user", parts=[Part.from_text("Say this is a test")] - ) - ], - generation_config=GenerationConfig(temperature=1000), - ) - except BadRequest: - pass - - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.request.temperature": 1000.0, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', - } - assert_span_error(spans[0]) - - -@pytest.mark.vcr() -def test_generate_content_invalid_role( - log_exporter: InMemoryLogRecordExporter, - generate_content: callable, - instrument_with_experimental_semconvs: VertexAIInstrumentor, -): - model = GenerativeModel("gemini-2.5-pro") - try: - # Fails because role must be "user" or "model" - generate_content( - model, - [ - Content( - role="invalid_role", - parts=[Part.from_text("Say this is a test")], - ) - ], - ) - except BadRequest: - pass - - # Emits the faulty content which caused the request to fail - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - log = logs[0].log_record - assert log.attributes == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.input.messages": ( - { - "role": "invalid_role", - "parts": ({"type": "text", "content": "Say this is a test"},), - }, - ), - } - - -@pytest.mark.vcr() -def test_generate_content_extra_params( - span_exporter, - instrument_no_content_with_experimental_semconvs, - generate_content: callable, -): - generation_config = GenerationConfig( - top_k=2, - top_p=0.95, - temperature=0.2, - stop_sequences=["\n\n\n"], - max_output_tokens=5, - presence_penalty=-1.5, - frequency_penalty=1.0, - seed=12345, - ) - model = GenerativeModel("gemini-2.5-pro") - generate_content( - model, - [ - Content(role="user", parts=[Part.from_text("Say this is a test")]), - ], - generation_config=generation_config, - ) - - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.frequency_penalty": 1.0, - "gen_ai.request.max_tokens": 5, - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.request.presence_penalty": -1.5, - "gen_ai.request.stop_sequences": ("\n\n\n",), - "gen_ai.request.temperature": 0.20000000298023224, - "gen_ai.request.top_p": 0.949999988079071, - "gen_ai.response.finish_reasons": ("length",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.request.seed": 12345, - "gen_ai.usage.input_tokens": 5, - "gen_ai.usage.output_tokens": 0, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } - - -def assert_span_error(span: ReadableSpan) -> None: - # Sets error status - assert span.status.status_code == StatusCode.ERROR - - # TODO: check thate error.type is set - # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md - - # Records exception event - error_events = [e for e in span.events if e.name == "exception"] - assert error_events != [] - - -@pytest.mark.vcr() -def test_generate_content_all_events( - log_exporter: InMemoryLogRecordExporter, - generate_content: callable, - instrument_with_experimental_semconvs: VertexAIInstrumentor, -): - generate_content_all_input_events( - GenerativeModel( - "gemini-2.5-pro", - system_instruction=Part.from_text( - "You are a clever language model" - ), - ), - log_exporter, - instrument_with_experimental_semconvs, - ) - - -@pytest.mark.vcr() -def test_preview_generate_content_all_input_events( - log_exporter: InMemoryLogRecordExporter, - generate_content: callable, - instrument_with_experimental_semconvs: VertexAIInstrumentor, -): - generate_content_all_input_events( - PreviewGenerativeModel( - "gemini-2.5-pro", - system_instruction=Part.from_text( - "You are a clever language model" - ), - ), - log_exporter, - instrument_with_experimental_semconvs, - ) - - -def generate_content_all_input_events( - model: GenerativeModel | PreviewGenerativeModel, - log_exporter: InMemoryLogRecordExporter, - instrument_with_experimental_semconvs: VertexAIInstrumentor, -): - model.generate_content( - [ - Content( - role="user", parts=[Part.from_text("My name is OpenTelemetry")] - ), - Content( - role="model", parts=[Part.from_text("Hello OpenTelemetry!")] - ), - Content( - role="user", - parts=[ - Part.from_text("Address me by name and say this is a test") - ], - ), - ], - generation_config=GenerationConfig( - seed=12345, response_mime_type="text/plain" - ), - ) - # Emits a single log. - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - log = logs[0].log_record - assert log.attributes == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.request.seed": 12345, - "gen_ai.output.type": "text", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.usage.input_tokens": 25, - "gen_ai.usage.output_tokens": 8, - "gen_ai.system_instructions": ( - {"type": "text", "content": "You are a clever language model"}, - ), - "gen_ai.input.messages": ( - { - "role": "user", - "parts": ( - { - "type": "text", - "content": "My name is OpenTelemetry", - }, - ), - }, - { - "role": "model", - "parts": ( - {"type": "text", "content": "Hello OpenTelemetry!"}, - ), - }, - { - "role": "user", - "parts": ( - { - "type": "text", - "content": "Address me by name and say this is a test", - }, - ), - }, - ), - "gen_ai.output.messages": ( - { - "role": "model", - "parts": ( - { - "type": "text", - "content": "OpenTelemetry, this is a test.", - }, - ), - "finish_reason": "stop", - }, - ), - } diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling.py index 74269d69..8837c0cf 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling.py @@ -1,3 +1,5 @@ +import json + import pytest from tests.shared_test_utils import ( ask_about_weather, @@ -21,6 +23,21 @@ ) +def assert_handler_event(log, parent_span): + """Assert log record is the unified GenAI handler event and return its body.""" + assert ( + log.log_record.event_name + == "gen_ai.client.inference.operation.details" + ) + assert log.log_record.body is not None + if parent_span: + span_context = parent_span.get_span_context() + assert log.log_record.trace_id == span_context.trace_id + assert log.log_record.span_id == span_context.span_id + assert log.log_record.trace_flags == span_context.trace_flags + return dict(log.log_record.body) + + @pytest.mark.vcr() def test_function_call_choice( span_exporter: InMemorySpanExporter, @@ -33,112 +50,71 @@ def test_function_call_choice( # Emits span spans = span_exporter.get_finished_spans() assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", - "gen_ai.usage.input_tokens": 74, - "gen_ai.usage.output_tokens": 16, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } - - # Emits user and choice events - logs = log_exporter.get_finished_logs() - assert len(logs) == 2 - user_log, choice_log = [log_data.log_record for log_data in logs] - assert user_log.attributes == {"gen_ai.system": "vertex_ai"} - assert user_log.event_name == "gen_ai.user.message" - assert user_log.body == { - "content": [ - {"text": "Get weather details in New Delhi and San Francisco?"} - ], - "role": "user", - } - - assert choice_log.attributes == {"gen_ai.system": "vertex_ai"} - assert choice_log.event_name == "gen_ai.choice" - assert choice_log.body == { - "finish_reason": "stop", - "index": 0, - "message": { - "content": [ + span = spans[0] + assert span.name == "chat gemini-2.5-pro" + attrs = dict(span.attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.response.finish_reasons"] == ("stop",) + assert attrs["gen_ai.response.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert attrs["gen_ai.usage.input_tokens"] == 74 + assert attrs["gen_ai.usage.output_tokens"] == 16 + assert attrs["server.address"] == "us-central1-aiplatform.googleapis.com" + assert attrs["server.port"] == 443 + + # Content on span + assert "gen_ai.input.messages" in attrs + input_msgs = json.loads(attrs["gen_ai.input.messages"]) + assert input_msgs == [ + { + "role": "user", + "parts": [ { - "function_call": { - "args": {"location": "New Delhi"}, - "name": "get_current_weather", - } - }, - { - "function_call": { - "args": {"location": "San Francisco"}, - "name": "get_current_weather", - } - }, + "type": "text", + "content": "Get weather details in New Delhi and San Francisco?", + } ], - "role": "model", - }, - "tool_calls": [ - { - "function": { - "arguments": {"location": "New Delhi"}, - "name": "get_current_weather", - }, - "id": "get_current_weather_0", - "type": "function", - }, - { - "function": { - "arguments": {"location": "San Francisco"}, - "name": "get_current_weather", - }, - "id": "get_current_weather_1", - "type": "function", - }, - ], - } + } + ] + + # Output messages on span - function_call parts are skipped (HYBIM-604) + assert "gen_ai.output.messages" in attrs + output_msgs = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_msgs) == 1 + assert output_msgs[0]["role"] == "model" + assert output_msgs[0]["finish_reason"] == "stop" + + # Content events emitter emits a single event + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + body = assert_handler_event(logs[0], span) + assert "gen_ai.input.messages" in body + assert "gen_ai.output.messages" in body @pytest.mark.vcr() def test_function_call_choice_no_content( + span_exporter: InMemorySpanExporter, log_exporter: InMemoryLogRecordExporter, instrument_no_content: VertexAIInstrumentor, generate_content: callable, ): ask_about_weather(generate_content) - # Emits user and choice events + # Emits span without content + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + attrs = dict(spans[0].attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert "gen_ai.input.messages" not in attrs + assert "gen_ai.output.messages" not in attrs + + # No events emitted when content is disabled logs = log_exporter.get_finished_logs() - assert len(logs) == 2 - user_log, choice_log = [log_data.log_record for log_data in logs] - assert user_log.attributes == {"gen_ai.system": "vertex_ai"} - assert user_log.event_name == "gen_ai.user.message" - assert user_log.body == { - "role": "user", - } - - assert choice_log.attributes == {"gen_ai.system": "vertex_ai"} - assert choice_log.event_name == "gen_ai.choice" - assert choice_log.body == { - "finish_reason": "stop", - "index": 0, - "message": {"role": "model"}, - "tool_calls": [ - { - "function": {"name": "get_current_weather"}, - "id": "get_current_weather_0", - "type": "function", - }, - { - "function": {"name": "get_current_weather"}, - "id": "get_current_weather_1", - "type": "function", - }, - ], - } + assert len(logs) == 0 @pytest.mark.vcr() @@ -153,84 +129,55 @@ def test_tool_events( # Emits span spans = span_exporter.get_finished_spans() assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", - "gen_ai.usage.input_tokens": 128, - "gen_ai.usage.output_tokens": 26, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } - logs = log_exporter.get_finished_logs() - # Emits user, assistant, two tool, and choice events - assert len(logs) == 5 - user_log, assistant_log, tool_log1, tool_log2, choice_log = [ - log_data.log_record for log_data in logs + span = spans[0] + assert span.name == "chat gemini-2.5-pro" + attrs = dict(span.attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.response.finish_reasons"] == ("stop",) + assert attrs["gen_ai.response.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert attrs["gen_ai.usage.input_tokens"] == 128 + assert attrs["gen_ai.usage.output_tokens"] == 26 + assert attrs["server.address"] == "us-central1-aiplatform.googleapis.com" + assert attrs["server.port"] == 443 + + # Content on span: user text, model function_call (skipped), user tool responses, model text response + assert "gen_ai.input.messages" in attrs + input_msgs = json.loads(attrs["gen_ai.input.messages"]) + assert len(input_msgs) == 3 + # First message: user text + assert input_msgs[0]["role"] == "user" + assert input_msgs[0]["parts"] == [ + { + "type": "text", + "content": "Get weather details in New Delhi and San Francisco?", + } ] - assert user_log.attributes == {"gen_ai.system": "vertex_ai"} - assert user_log.event_name == "gen_ai.user.message" - assert user_log.body == { - "content": [ - {"text": "Get weather details in New Delhi and San Francisco?"} - ], - "role": "user", - } - - assert assistant_log.attributes == {"gen_ai.system": "vertex_ai"} - assert assistant_log.event_name == "gen_ai.assistant.message" - assert assistant_log.body == { - "role": "model", - "content": [ - { - "function_call": { - "name": "get_current_weather", - "args": {"location": "New Delhi"}, - } - }, - { - "function_call": { - "name": "get_current_weather", - "args": {"location": "San Francisco"}, - } - }, - ], - } - - assert tool_log1.attributes == {"gen_ai.system": "vertex_ai"} - assert tool_log1.event_name == "gen_ai.tool.message" - - assert tool_log1.body == { - "role": "user", - "id": "get_current_weather_0", - "content": {"content": '{"temperature": 35, "unit": "C"}'}, - } - - assert tool_log2.attributes == {"gen_ai.system": "vertex_ai"} - assert tool_log2.event_name == "gen_ai.tool.message" - assert tool_log2.body == { - "role": "user", - "id": "get_current_weather_1", - "content": {"content": '{"temperature": 25, "unit": "C"}'}, - } - - assert choice_log.attributes == {"gen_ai.system": "vertex_ai"} - assert choice_log.event_name == "gen_ai.choice" - assert choice_log.body == { - "finish_reason": "stop", - "index": 0, - "message": { - "content": [ - { - "text": "The current temperature in New Delhi is 35°C, and in San Francisco, it is 25°C." - } - ], - "role": "model", - }, - } + # Second message: model with function_call parts (skipped by convert_content_to_message_parts) + assert input_msgs[1]["role"] == "model" + assert input_msgs[1]["parts"] == [] + # Third message: user with tool call responses + assert input_msgs[2]["role"] == "user" + assert len(input_msgs[2]["parts"]) == 2 + assert input_msgs[2]["parts"][0]["type"] == "tool_call_response" + assert input_msgs[2]["parts"][1]["type"] == "tool_call_response" + + # Output messages on span + assert "gen_ai.output.messages" in attrs + output_msgs = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_msgs) == 1 + assert output_msgs[0]["role"] == "model" + assert output_msgs[0]["finish_reason"] == "stop" + assert len(output_msgs[0]["parts"]) == 1 + assert output_msgs[0]["parts"][0]["type"] == "text" + + # Content events emitter emits a single event + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + body = assert_handler_event(logs[0], span) + assert "gen_ai.input.messages" in body + assert "gen_ai.output.messages" in body @pytest.mark.vcr() @@ -242,57 +189,22 @@ def test_tool_events_no_content( ): ask_about_weather_function_response(generate_content) - # Emits span + # Emits span without content spans = span_exporter.get_finished_spans() assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", - "gen_ai.usage.input_tokens": 128, - "gen_ai.usage.output_tokens": 22, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } + attrs = dict(spans[0].attributes) + assert attrs["gen_ai.operation.name"] == "chat" + assert attrs["gen_ai.request.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.response.finish_reasons"] == ("stop",) + assert attrs["gen_ai.response.model"] == "gemini-2.5-pro" + assert attrs["gen_ai.provider.name"] == "vertex_ai" + assert attrs["gen_ai.usage.input_tokens"] == 128 + assert attrs["gen_ai.usage.output_tokens"] == 22 + assert attrs["server.address"] == "us-central1-aiplatform.googleapis.com" + assert attrs["server.port"] == 443 + assert "gen_ai.input.messages" not in attrs + assert "gen_ai.output.messages" not in attrs + + # No events emitted when content is disabled logs = log_exporter.get_finished_logs() - # Emits user, assistant, two tool, and choice events - assert len(logs) == 5 - user_log, assistant_log, tool_log1, tool_log2, choice_log = [ - log_data.log_record for log_data in logs - ] - assert user_log.attributes == {"gen_ai.system": "vertex_ai"} - assert user_log.event_name == "gen_ai.user.message" - assert user_log.body == {"role": "user"} - - assert assistant_log.attributes == {"gen_ai.system": "vertex_ai"} - assert assistant_log.event_name == "gen_ai.assistant.message" - assert assistant_log.body == {"role": "model"} - - assert tool_log1.attributes == { - "gen_ai.system": "vertex_ai", - } - assert tool_log1.event_name == "gen_ai.tool.message" - assert tool_log1.body == { - "role": "user", - "id": "get_current_weather_0", - } - assert tool_log1.event_name == "gen_ai.tool.message" - - assert tool_log2.attributes == {"gen_ai.system": "vertex_ai"} - - assert tool_log2.body == { - "role": "user", - "id": "get_current_weather_1", - } - assert tool_log2.event_name == "gen_ai.tool.message" - - assert choice_log.attributes == {"gen_ai.system": "vertex_ai"} - assert choice_log.event_name == "gen_ai.choice" - assert choice_log.body == { - "finish_reason": "stop", - "index": 0, - "message": {"role": "model"}, - } + assert len(logs) == 0 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py deleted file mode 100644 index e2513630..00000000 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py +++ /dev/null @@ -1,357 +0,0 @@ -import json -import time -from typing import Any - -import fsspec -import pytest -from tests.shared_test_utils import ( - ask_about_weather, - ask_about_weather_function_response, -) - -from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor - -# Backward compatibility for InMemoryLogExporter -> InMemoryLogRecordExporter rename -try: - from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import ( # pylint: disable=no-name-in-module - InMemoryLogRecordExporter, - ) -except ImportError: - # Fallback to old name for compatibility with older SDK versions - from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import ( - InMemoryLogExporter as InMemoryLogRecordExporter, - ) -from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( - InMemorySpanExporter, -) - - -def test_function_call_choice( - span_exporter: InMemorySpanExporter, - log_exporter: InMemoryLogRecordExporter, - instrument_with_experimental_semconvs: VertexAIInstrumentor, - generate_content: callable, -): - ask_about_weather(generate_content) - - # Emits span - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.usage.input_tokens": 74, - "gen_ai.usage.output_tokens": 16, - "server.address": "us-central1-aiplatform.googleapis.com", - "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Get weather details in New Delhi and San Francisco?","type":"text"}]}]', - "gen_ai.output.messages": '[{"role":"model","parts":[{"arguments":{"location":"New Delhi"},"name":"get_current_weather","id":"get_current_weather_0","type":"tool_call"},{"arguments":{"location":"San Francisco"},"name":"get_current_weather","id":"get_current_weather_1","type":"tool_call"}],"finish_reason":"stop"}]', - "server.port": 443, - } - - # Emits user and choice events - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - log = logs[0].log_record - assert log.attributes == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.usage.input_tokens": 74, - "gen_ai.usage.output_tokens": 16, - "gen_ai.input.messages": ( - { - "role": "user", - "parts": ( - { - "type": "text", - "content": "Get weather details in New Delhi and San Francisco?", - }, - ), - }, - ), - "gen_ai.output.messages": ( - { - "role": "model", - "parts": ( - { - "type": "tool_call", - "arguments": {"location": "New Delhi"}, - "name": "get_current_weather", - "id": "get_current_weather_0", - }, - { - "type": "tool_call", - "arguments": {"location": "San Francisco"}, - "name": "get_current_weather", - "id": "get_current_weather_1", - }, - ), - "finish_reason": "stop", - }, - ), - } - - -@pytest.mark.vcr() -def test_function_call_choice_no_content( - log_exporter: InMemoryLogRecordExporter, - instrument_no_content_with_experimental_semconvs: VertexAIInstrumentor, - generate_content: callable, -): - ask_about_weather(generate_content) - - # Emits user and choice events - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - log = logs[0].log_record - assert log.attributes == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.usage.input_tokens": 74, - "gen_ai.usage.output_tokens": 16, - } - - -@pytest.mark.vcr() -def test_tool_events( - span_exporter: InMemorySpanExporter, - log_exporter: InMemoryLogRecordExporter, - instrument_with_experimental_semconvs: VertexAIInstrumentor, - generate_content: callable, -): - ask_about_weather_function_response(generate_content) - - # Emits span - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.usage.input_tokens": 128, - "gen_ai.usage.output_tokens": 26, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Get weather details in New Delhi and San Francisco?","type":"text"}]},{"role":"model","parts":[{"arguments":{"location":"New Delhi"},"name":"get_current_weather","id":"get_current_weather_0","type":"tool_call"},{"arguments":{"location":"San Francisco"},"name":"get_current_weather","id":"get_current_weather_1","type":"tool_call"}]},{"role":"user","parts":[{"response":{"content":"{\\"temperature\\": 35, \\"unit\\": \\"C\\"}"},"id":"get_current_weather_0","type":"tool_call_response"},{"response":{"content":"{\\"temperature\\": 25, \\"unit\\": \\"C\\"}"},"id":"get_current_weather_1","type":"tool_call_response"}]}]', - "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"The current temperature in New Delhi is 35\\u00b0C, and in San Francisco, it is 25\\u00b0C.","type":"text"}],"finish_reason":"stop"}]', - } - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - log = logs[0].log_record - assert log.attributes == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.usage.input_tokens": 128, - "gen_ai.usage.output_tokens": 26, - "gen_ai.input.messages": ( - { - "role": "user", - "parts": ( - { - "type": "text", - "content": "Get weather details in New Delhi and San Francisco?", - }, - ), - }, - { - "role": "model", - "parts": ( - { - "type": "tool_call", - "arguments": {"location": "New Delhi"}, - "name": "get_current_weather", - "id": "get_current_weather_0", - }, - { - "type": "tool_call", - "arguments": {"location": "San Francisco"}, - "name": "get_current_weather", - "id": "get_current_weather_1", - }, - ), - }, - { - "role": "user", - "parts": ( - { - "type": "tool_call_response", - "response": { - "content": '{"temperature": 35, "unit": "C"}' - }, - "id": "get_current_weather_0", - }, - { - "type": "tool_call_response", - "response": { - "content": '{"temperature": 25, "unit": "C"}' - }, - "id": "get_current_weather_1", - }, - ), - }, - ), - "gen_ai.output.messages": ( - { - "role": "model", - "parts": ( - { - "type": "text", - "content": "The current temperature in New Delhi is 35°C, and in San Francisco, it is 25°C.", - }, - ), - "finish_reason": "stop", - }, - ), - } - - -@pytest.mark.vcr() -def test_tool_events_no_content( - span_exporter: InMemorySpanExporter, - log_exporter: InMemoryLogRecordExporter, - instrument_no_content_with_experimental_semconvs: VertexAIInstrumentor, - generate_content: callable, -): - ask_about_weather_function_response(generate_content) - - # Emits span - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "chat gemini-2.5-pro" - assert dict(spans[0].attributes) == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.usage.input_tokens": 128, - "gen_ai.usage.output_tokens": 22, - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - } - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - log = logs[0].log_record - assert log.attributes == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", - "server.address": "us-central1-aiplatform.googleapis.com", - "server.port": 443, - "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.usage.input_tokens": 128, - "gen_ai.usage.output_tokens": 22, - } - - -def assert_fsspec_equal(path: str, value: Any) -> None: - # Hide this function and its calls from traceback. - __tracebackhide__ = True # pylint: disable=unused-variable - with fsspec.open(path, "r") as file: - assert json.load(file) == value - - -@pytest.mark.vcr() -def test_tool_events_with_completion_hook( - span_exporter: InMemorySpanExporter, - log_exporter: InMemoryLogRecordExporter, - instrument_with_upload_hook: VertexAIInstrumentor, - generate_content: callable, -): - ask_about_weather_function_response(generate_content) - - # Emits span - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - logs = log_exporter.get_finished_logs() - assert len(logs) == 1 - # File upload takes a few seconds sometimes. - time.sleep(3) - - # Both log and span have the same reference attributes from upload hook - for key in "gen_ai.input.messages_ref", "gen_ai.output.messages_ref": - assert spans[0].attributes.get(key) - assert logs[0].log_record.attributes.get(key) - - assert spans[0].attributes[key] == logs[0].log_record.attributes[key] - - assert_fsspec_equal( - spans[0].attributes["gen_ai.output.messages_ref"], - [ - { - "role": "model", - "parts": [ - { - "content": "The weather in New Delhi is 35°C and in San Francisco is 25°C.", - "type": "text", - } - ], - "finish_reason": "stop", - } - ], - ) - assert_fsspec_equal( - spans[0].attributes["gen_ai.input.messages_ref"], - [ - { - "parts": [ - { - "content": "Get weather details in New Delhi and San Francisco?", - "type": "text", - } - ], - "role": "user", - }, - { - "parts": [ - { - "arguments": {"location": "New Delhi"}, - "id": "get_current_weather_0", - "name": "get_current_weather", - "type": "tool_call", - }, - { - "arguments": {"location": "San Francisco"}, - "id": "get_current_weather_1", - "name": "get_current_weather", - "type": "tool_call", - }, - ], - "role": "model", - }, - { - "parts": [ - { - "id": "get_current_weather_0", - "response": { - "content": '{"temperature": 35, "unit": "C"}' - }, - "type": "tool_call_response", - }, - { - "id": "get_current_weather_1", - "response": { - "content": '{"temperature": 25, "unit": "C"}' - }, - "type": "tool_call_response", - }, - ], - "role": "user", - }, - ], - )