From c1e58a5e5a3eb0a2e780877e85c9a2fa39f85f9d Mon Sep 17 00:00:00 2001 From: wangzlei Date: Tue, 21 Apr 2026 15:48:37 -0700 Subject: [PATCH 1/2] Remove is_aws_agentic_observability_opt_in, restore v0.15 legacy is_agent_observability_enabled --- .../src/amazon/opentelemetry/distro/_utils.py | 15 +-- .../distro/aws_opentelemetry_configurator.py | 14 +-- .../distro/aws_opentelemetry_distro.py | 104 ++++++--------- .../otlp/aws/common/_aws_http_headers.py | 4 +- .../otlp/aws/traces/otlp_aws_span_exporter.py | 6 +- .../distro/patches/_starlette_patches.py | 4 +- .../aws/traces/test_otlp_aws_span_exporter.py | 14 +-- .../distro/patches/test_starlette_patches.py | 2 - .../test_aws_opentelementry_configurator.py | 12 +- .../distro/test_aws_opentelemetry_distro.py | 119 ++---------------- .../amazon/opentelemetry/distro/test_utils.py | 38 ------ 11 files changed, 76 insertions(+), 256 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py index 91cbcf93c..bd5f2a367 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py @@ -10,12 +10,9 @@ _logger: Logger = getLogger(__name__) -# Maintained for backwards compatibility. New users should use AWS_AGENTIC_OBSERVABILITY_OPT_IN instead. AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED" -AWS_AGENTIC_OBSERVABILITY_OPT_IN = "AWS_AGENTIC_OBSERVABILITY_OPT_IN" OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS = "OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS" - def is_installed(req: str) -> bool: """Is the given required package installed?""" req = Requirement(req) @@ -37,20 +34,10 @@ def is_installed(req: str) -> bool: def is_agent_observability_enabled() -> bool: - # Maintained for backwards compatibility. New users should use AWS_AGENTIC_OBSERVABILITY_OPT_IN instead. + """Is the Agentic AI monitoring flag set to true?""" return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true" -def is_aws_agentic_observability_opt_in() -> bool: - """Is the AI observability opt-in flag set to true?""" - return os.environ.get(AWS_AGENTIC_OBSERVABILITY_OPT_IN, "false").lower() == "true" - - -def is_agentic_observability_enabled() -> bool: - """Returns True if either AGENT_OBSERVABILITY_ENABLED or AWS_AGENTIC_OBSERVABILITY_OPT_IN is set to true.""" - return is_agent_observability_enabled() or is_aws_agentic_observability_opt_in() - - def should_add_application_signals_dimensions() -> bool: """Should Service and Environment Application Signals dimensions be added to EMF logs?""" return os.environ.get(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "true").lower() == "true" diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py index b89c8eb6d..81f87ba3b 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py @@ -17,7 +17,7 @@ from amazon.opentelemetry.distro._aws_attribute_keys import AWS_LOCAL_SERVICE, AWS_SERVICE_TYPE from amazon.opentelemetry.distro._aws_resource_attribute_configurator import get_service_attribute -from amazon.opentelemetry.distro._utils import get_aws_session, is_agentic_observability_enabled +from amazon.opentelemetry.distro._utils import get_aws_session, is_agent_observability_enabled from amazon.opentelemetry.distro.always_record_sampler import AlwaysRecordSampler from amazon.opentelemetry.distro.attribute_propagating_span_processor_builder import ( AttributePropagatingSpanProcessorBuilder, @@ -205,7 +205,7 @@ def _initialize_components(): AwsEksResourceDetector(), AwsEcsResourceDetector(), ] - if not (_is_lambda_environment() or is_agentic_observability_enabled()) + if not (_is_lambda_environment() or is_agent_observability_enabled()) else [] ) @@ -327,7 +327,7 @@ def _export_unsampled_span_for_lambda(trace_provider: TracerProvider, resource: def _export_unsampled_span_for_agent_observability(trace_provider: TracerProvider, resource: Resource = None): - if not is_agentic_observability_enabled(): + if not is_agent_observability_enabled(): return traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) @@ -468,7 +468,7 @@ def _customize_log_record_processor(logger_provider: LoggerProvider, log_exporte if not log_exporter: return - if is_agentic_observability_enabled(): + if is_agent_observability_enabled(): # pylint: disable=import-outside-toplevel from amazon.opentelemetry.distro.exporter.otlp.aws.logs._aws_cw_otlp_batch_log_record_processor import ( AwsCloudWatchOtlpBatchLogRecordProcessor, @@ -527,7 +527,7 @@ def _customize_span_processors(provider: TracerProvider, resource: Resource, sam # AI applications typically have low throughput traffic patterns and require # comprehensive monitoring to catch subtle failure modes like hallucinations # and quality degradation that sampling could miss. - if is_agentic_observability_enabled(): + if is_agent_observability_enabled(): _export_unsampled_span_for_agent_observability(provider, resource) provider.add_span_processor(GenAiNestedClientSpanProcessor()) baggage_keys.add("session.id") @@ -635,7 +635,7 @@ def _customize_resource(resource: Resource) -> Resource: custom_attributes = {AWS_LOCAL_SERVICE: service_name} - if is_agentic_observability_enabled(): + if is_agent_observability_enabled(): # Add aws.service.type if it doesn't exist in the resource if resource and resource.attributes.get(AWS_SERVICE_TYPE) is None: # Set a default agent type for AI agent observability @@ -921,7 +921,7 @@ def _create_aws_otlp_exporter(endpoint: str, service: str, region: str): from amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter import OTLPAwsSpanExporter if service == XRAY_SERVICE: - if is_agentic_observability_enabled(): + if is_agent_observability_enabled(): # Span exporter needs an instance of logger provider in ai agent # observability case because we need to split input/output prompts # from span attributes and send them to the logs pipeline per diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py index 020aa63b0..bef3fdcd4 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py @@ -70,7 +70,6 @@ def _check_otel_version_compatibility(): OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, get_aws_region, is_agent_observability_enabled, - is_aws_agentic_observability_opt_in, is_installed, ) from amazon.opentelemetry.distro.aws_opentelemetry_configurator import APPLICATION_SIGNALS_ENABLED_CONFIG @@ -95,28 +94,26 @@ def _check_otel_version_compatibility(): _OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED as OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, ) from opentelemetry.sdk.environment_variables import ( - OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + OTEL_TRACES_SAMPLER, ) -from opentelemetry.util._importlib_metadata import EntryPoint _logger: Logger = getLogger(__name__) # Suppress configurator warnings from auto-instrumentation _load._logger.setLevel(LEVELS.get(os.environ.get(OTEL_PYTHON_LOG_LEVEL, "error").lower(), ERROR)) -AGENT_OBSERVABILITY_DISABLED_INSTRUMENTATIONS = ( - "sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector," - "system_metrics,google-genai,jinja2,aws_crewai,aws_langchain,aws_llama-index,aws_mcp,aws_openai_agents" -) - -AWS_AGENTIC_OBSERVABILITY_DISABLED_INSTRUMENTATIONS = ( - "sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector," - "system_metrics,google-genai,jinja2,crewai,langchain,llama-index,llama_index,mcp,openai_agents" -) +_THIRDPARTY_TO_AWS_NATIVE = { + "crewai": "aws_crewai", + "langchain": "aws_langchain", + "llama-index": "aws_llama-index", + "llama_index": "aws_llama-index", + "mcp": "aws_mcp", + "openai_agents": "aws_openai_agents", +} class AwsOpenTelemetryDistro(OpenTelemetryDistro): @@ -192,66 +189,39 @@ def _configure(self, **kwargs): OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, "base2_exponential_bucket_histogram" ) - if is_aws_agentic_observability_opt_in(): - _logger.info("AWS Agentic Observability enabled.") - self._configure_common_agent_observability(AWS_AGENTIC_OBSERVABILITY_DISABLED_INSTRUMENTATIONS) - os.environ.setdefault(OTEL_METRICS_EXPORTER, "otlp") - os.environ.setdefault("CREWAI_DISABLE_TELEMETRY", "true") - elif is_agent_observability_enabled(): - # Maintained for backwards compatibility. New users should use AWS_AGENTIC_OBSERVABILITY_OPT_IN instead. - _logger.info( - "AGENT_OBSERVABILITY_ENABLED is set. Consider using AWS_AGENTIC_OBSERVABILITY_OPT_IN for ADOT Agentic Observability." - ) - self._configure_common_agent_observability(AGENT_OBSERVABILITY_DISABLED_INSTRUMENTATIONS) + if is_agent_observability_enabled(): + os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp") + os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp") os.environ.setdefault(OTEL_METRICS_EXPORTER, "awsemf") + os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true") + region = get_aws_region() - if not os.environ.get(OTEL_EXPORTER_OTLP_ENDPOINT): - if region: - os.environ.setdefault( - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, f"https://xray.{region}.amazonaws.com/v1/traces" - ) - os.environ.setdefault( - OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, f"https://logs.{region}.amazonaws.com/v1/logs" - ) - else: - _logger.warning( - "AWS region could not be determined. OTLP endpoints will not be automatically configured. " - "Please set AWS_REGION environment variable or configure OTLP endpoints manually." - ) + if region: + os.environ.setdefault( + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, f"https://xray.{region}.amazonaws.com/v1/traces" + ) + os.environ.setdefault(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, f"https://logs.{region}.amazonaws.com/v1/logs") + else: + _logger.warning( + "AWS region could not be determined. OTLP endpoints will not be automatically configured. " + "Please set AWS_REGION environment variable or configure OTLP endpoints manually." + ) + + os.environ.setdefault(OTEL_TRACES_SAMPLER, "parentbased_always_on") + os.environ.setdefault( + OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, + "http,sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector," + "urllib3,requests,system_metrics,google-genai,jinja2", + ) + os.environ.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true") + os.environ.setdefault(OTEL_PYTHON_LOG_CORRELATION, "true") + os.environ.setdefault(APPLICATION_SIGNALS_ENABLED_CONFIG, "false") + os.environ.setdefault(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "false") + os.environ.setdefault("CREWAI_DISABLE_TELEMETRY", "true") + super(AwsOpenTelemetryDistro, self)._configure() if kwargs.get("apply_patches", True): apply_instrumentation_patches() - def load_instrumentor(self, entry_point: EntryPoint, **kwargs): - if self._should_skip_instrumentor(entry_point): - return - super().load_instrumentor(entry_point, **kwargs) - - @staticmethod - def _should_skip_instrumentor(entry_point: EntryPoint) -> bool: - # Some third-party SDKs register the same entry point name as the upstream - # OTel packages that we depend on. For Agentic Observability legacy mode, skip our bundled - # OTel instrumentation so that existing third-party setups are not brokens. - if ( - is_agent_observability_enabled() - and not is_aws_agentic_observability_opt_in() - and entry_point.dist - and entry_point.name == "openai_agents" - and entry_point.dist.name == "opentelemetry-instrumentation-openai-agents-v2" - ): - return True - # TODO: add additional skip conditions here as needed - return False - - @staticmethod - def _configure_common_agent_observability(disabled_instrumentations: str) -> None: - os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true") - os.environ.setdefault(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, disabled_instrumentations) - os.environ.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true") - os.environ.setdefault(OTEL_PYTHON_LOG_CORRELATION, "true") - os.environ.setdefault(APPLICATION_SIGNALS_ENABLED_CONFIG, "false") - os.environ.setdefault(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "false") - os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp") - os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp") diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/common/_aws_http_headers.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/common/_aws_http_headers.py index d3e8abf6d..4f67cd9e8 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/common/_aws_http_headers.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/common/_aws_http_headers.py @@ -1,14 +1,14 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from amazon.opentelemetry.distro._utils import is_agentic_observability_enabled +from amazon.opentelemetry.distro._utils import is_agent_observability_enabled from amazon.opentelemetry.distro.version import __version__ def build_user_agent() -> str: user_agent = f"ADOT-OTLP-Exporter-Python/{__version__}" - if is_agentic_observability_enabled(): + if is_agent_observability_enabled(): user_agent = f"ADOT-GenAI-OTLP-Exporter-Python/{__version__}" return user_agent diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py index 25518f950..c740502f9 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py @@ -6,7 +6,7 @@ from botocore.session import Session -from amazon.opentelemetry.distro._utils import is_agentic_observability_enabled +from amazon.opentelemetry.distro._utils import is_agent_observability_enabled from amazon.opentelemetry.distro.exporter.otlp.aws.common._aws_http_headers import _OTLP_AWS_HTTP_HEADERS from amazon.opentelemetry.distro.exporter.otlp.aws.common.aws_auth_session import AwsAuthSession from amazon.opentelemetry.distro.llo_handler import LLOHandler @@ -61,7 +61,7 @@ def __init__( def _ensure_llo_handler(self): """Lazily initialize LLO handler when needed to avoid initialization order issues""" - if self._llo_handler is None and is_agentic_observability_enabled(): + if self._llo_handler is None and is_agent_observability_enabled(): if self._logger_provider is None: try: self._logger_provider = get_logger_provider() @@ -77,7 +77,7 @@ def _ensure_llo_handler(self): def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: try: - if is_agentic_observability_enabled() and self._ensure_llo_handler(): + if is_agent_observability_enabled() and self._ensure_llo_handler(): llo_processed_spans = self._llo_handler.process_spans(spans) return super().export(llo_processed_spans) except Exception: # pylint: disable=broad-exception-caught diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py index 31d9ba0e3..bf6c5f58d 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py @@ -3,7 +3,7 @@ # Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License. from logging import Logger, getLogger -from amazon.opentelemetry.distro._utils import is_agentic_observability_enabled +from amazon.opentelemetry.distro._utils import is_agent_observability_enabled _logger: Logger = getLogger(__name__) @@ -25,7 +25,7 @@ def _apply_starlette_instrumentation_patches() -> None: # # Issue for tracking a feature to customize this setting within Starlette: # https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3725 - if is_agentic_observability_enabled(): + if is_agent_observability_enabled(): original_init = OpenTelemetryMiddleware.__init__ def patched_init(self, app, **kwargs): diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py index 2ec3dcf0c..8f8631a9d 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py @@ -51,7 +51,7 @@ def test_aws_headers_applied(self): self.assertIn("User-Agent", exporter._session.headers) @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agentic_observability_enabled" + "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" ) def test_ensure_llo_handler_when_disabled(self, mock_is_enabled): # Test _ensure_llo_handler when agent observability is disabled @@ -67,7 +67,7 @@ def test_ensure_llo_handler_when_disabled(self, mock_is_enabled): @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agentic_observability_enabled" + "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" ) @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler") def test_ensure_llo_handler_lazy_initialization( @@ -103,7 +103,7 @@ def test_ensure_llo_handler_lazy_initialization( @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agentic_observability_enabled" + "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" ) def test_ensure_llo_handler_with_existing_logger_provider(self, mock_is_enabled, mock_get_logger_provider): # Test when logger_provider is already provided @@ -130,7 +130,7 @@ def test_ensure_llo_handler_with_existing_logger_provider(self, mock_is_enabled, @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agentic_observability_enabled" + "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" ) def test_ensure_llo_handler_get_logger_provider_fails(self, mock_is_enabled, mock_get_logger_provider): # Test when get_logger_provider raises exception @@ -146,7 +146,7 @@ def test_ensure_llo_handler_get_logger_provider_fails(self, mock_is_enabled, moc self.assertIsNone(exporter._llo_handler) @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agentic_observability_enabled" + "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" ) def test_export_with_llo_disabled(self, mock_is_enabled): # Test export when LLO is disabled @@ -167,7 +167,7 @@ def test_export_with_llo_disabled(self, mock_is_enabled): self.assertIsNone(exporter._llo_handler) @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agentic_observability_enabled" + "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" ) @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler") @@ -199,7 +199,7 @@ def test_export_with_llo_enabled(self, mock_llo_handler_class, mock_get_logger_p mock_parent_export.assert_called_once_with(processed_spans) @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agentic_observability_enabled" + "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" ) @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler") diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py index 70cb00f68..d90a31457 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_starlette_patches.py @@ -23,9 +23,7 @@ def test_starlette_patch_applied_successfully(self, mock_logger): """Test that the Starlette ASGI middleware patch is applied successfully.""" env_configs = [ {"AGENT_OBSERVABILITY_ENABLED": "true"}, - {"AWS_AGENTIC_OBSERVABILITY_OPT_IN": "true"}, {"AGENT_OBSERVABILITY_ENABLED": "false"}, - {"AWS_AGENTIC_OBSERVABILITY_OPT_IN": "false"}, ] for env_vars in env_configs: agent_enabled = any(v == "true" for v in env_vars.values()) diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py index 07babd32b..265a341a2 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py @@ -1304,7 +1304,7 @@ def test_fetch_logs_header(self): _clear_logs_header_cache() @patch( - "amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agentic_observability_enabled", + "amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agent_observability_enabled", return_value=False, ) def test_customize_log_record_processor_without_agent_observability(self, _): @@ -1319,7 +1319,7 @@ def test_customize_log_record_processor_without_agent_observability(self, _): self.assertIsInstance(added_processor, BatchLogRecordProcessor) @patch( - "amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agentic_observability_enabled", return_value=True + "amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agent_observability_enabled", return_value=True ) def test_customize_log_record_processor_with_agent_observability(self, _): """Test that AwsCloudWatchOtlpBatchLogRecordProcessor is used when agent observability is enabled""" @@ -1333,7 +1333,7 @@ def test_customize_log_record_processor_with_agent_observability(self, _): self.assertIsInstance(added_processor, AwsCloudWatchOtlpBatchLogRecordProcessor) @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.get_logger_provider") - @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agentic_observability_enabled") + @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agent_observability_enabled") @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.get_aws_session") def test_create_aws_otlp_exporter(self, mock_get_session, mock_is_agent_enabled, mock_get_logger_provider): # Test when botocore is not installed @@ -1398,7 +1398,7 @@ def test_create_aws_otlp_exporter(self, mock_get_session, mock_is_agent_enabled, result = _create_aws_otlp_exporter("https://xray.us-east-1.amazonaws.com/v1/traces", "xray", "us-east-1") self.assertIsNone(result) - @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agentic_observability_enabled") + @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agent_observability_enabled") @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.get_service_attribute") def test_customize_resource_without_agent_observability(self, mock_get_service_attribute, mock_is_agent_enabled): """Test _customize_resource when agent observability is disabled""" @@ -1412,7 +1412,7 @@ def test_customize_resource_without_agent_observability(self, mock_get_service_a self.assertEqual(result.attributes[AWS_LOCAL_SERVICE], "test-service") self.assertNotIn(AWS_SERVICE_TYPE, result.attributes) - @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agentic_observability_enabled") + @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agent_observability_enabled") @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.get_service_attribute") def test_customize_resource_with_agent_observability_default( self, mock_get_service_attribute, mock_is_agent_enabled @@ -1428,7 +1428,7 @@ def test_customize_resource_with_agent_observability_default( self.assertEqual(result.attributes[AWS_LOCAL_SERVICE], "test-service") self.assertEqual(result.attributes[AWS_SERVICE_TYPE], "gen_ai_agent") - @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agentic_observability_enabled") + @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.is_agent_observability_enabled") @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator.get_service_attribute") def test_customize_resource_with_existing_agent_type(self, mock_get_service_attribute, mock_is_agent_enabled): """Test _customize_resource when agent type already exists in resource""" diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelemetry_distro.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelemetry_distro.py index f7c178541..5f80b750c 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelemetry_distro.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelemetry_distro.py @@ -10,7 +10,6 @@ from amazon.opentelemetry.distro.aws_opentelemetry_configurator import APPLICATION_SIGNALS_ENABLED_CONFIG from amazon.opentelemetry.distro.aws_opentelemetry_distro import ( - AGENT_OBSERVABILITY_DISABLED_INSTRUMENTATIONS, AwsOpenTelemetryDistro, ) from opentelemetry import propagate @@ -60,7 +59,6 @@ def setUp(self): "DJANGO_SETTINGS_MODULE", OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, - "AWS_AGENTIC_OBSERVABILITY_OPT_IN", ] # First, save all current values @@ -160,10 +158,10 @@ def test_configure_with_agent_observability_enabled( os.environ.get(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT), "https://logs.us-west-2.amazonaws.com/v1/logs" ) - self.assertEqual( - os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS), - AGENT_OBSERVABILITY_DISABLED_INSTRUMENTATIONS, - ) + disabled = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "") + self.assertIn("system_metrics", disabled) + self.assertIn("google-genai", disabled) + self.assertIn("jinja2", disabled) self.assertEqual(os.environ.get(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED), "true") self.assertEqual(os.environ.get(APPLICATION_SIGNALS_ENABLED_CONFIG), "false") self.assertEqual(os.environ.get("OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS"), "false") @@ -272,7 +270,6 @@ def test_configure_with_agent_observability_endpoints_already_set( @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.get_aws_region") @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_agent_observability_enabled") - @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_aws_agentic_observability_opt_in") @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_installed") @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches") @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure") @@ -281,13 +278,11 @@ def test_configure_agent_observability_v1( mock_super_configure, mock_apply_patches, mock_is_installed, - mock_is_aws_agentic_observability_opt_in, mock_is_agent_observability, mock_get_aws_region, ): - """Test that AGENT_OBSERVABILITY_ENABLED uses v1 configuration""" + """Test that AGENT_OBSERVABILITY_ENABLED uses v0.15 configuration""" mock_is_agent_observability.return_value = True - mock_is_aws_agentic_observability_opt_in.return_value = False mock_get_aws_region.return_value = "us-east-1" mock_is_installed.return_value = False @@ -305,58 +300,11 @@ def test_configure_agent_observability_v1( self.assertEqual(os.environ.get(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED), "true") self.assertEqual(os.environ.get(APPLICATION_SIGNALS_ENABLED_CONFIG), "false") self.assertEqual(os.environ.get("OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS"), "false") + self.assertEqual(os.environ.get("CREWAI_DISABLE_TELEMETRY"), "true") disabled = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "").split(",") - self.assertNotIn("crewai", disabled) - self.assertNotIn("langchain", disabled) - self.assertNotIn("llama-index", disabled) - self.assertNotIn("llama_index", disabled) - self.assertNotIn("mcp", disabled) - self.assertIn("jinja2", disabled) - self.assertIn("aws_crewai", disabled) - self.assertIn("aws_langchain", disabled) - self.assertIn("aws_llama-index", disabled) - self.assertIn("aws_mcp", disabled) - - @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.get_aws_region") - @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_agent_observability_enabled") - @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_aws_agentic_observability_opt_in") - @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_installed") - @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches") - @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure") - def test_configure_ai_observability_opt_in( - self, - mock_super_configure, - mock_apply_patches, - mock_is_installed, - mock_is_aws_agentic_observability_opt_in, - mock_is_agent_observability, - mock_get_aws_region, - ): - """Test that AI_OBSERVABILITY_OPT_IN uses v2 collector config and disables 3rd party instrumentations""" - mock_is_agent_observability.return_value = False - mock_is_aws_agentic_observability_opt_in.return_value = True - mock_get_aws_region.return_value = "us-east-1" - mock_is_installed.return_value = False - - AwsOpenTelemetryDistro()._configure() - - self.assertEqual(os.environ.get(OTEL_METRICS_EXPORTER), "otlp") - self.assertEqual(os.environ.get("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"), "true") - - self.assertEqual(os.environ.get(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED), "true") - self.assertEqual(os.environ.get(APPLICATION_SIGNALS_ENABLED_CONFIG), "false") - self.assertEqual(os.environ.get("OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS"), "false") - disabled = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "").split(",") - self.assertIn("crewai", disabled) - self.assertIn("langchain", disabled) - self.assertIn("llama-index", disabled) - self.assertIn("llama_index", disabled) - self.assertIn("mcp", disabled) + self.assertIn("system_metrics", disabled) + self.assertIn("google-genai", disabled) self.assertIn("jinja2", disabled) - self.assertNotIn("aws_crewai", disabled) - self.assertNotIn("aws_langchain", disabled) - self.assertNotIn("aws_llama-index", disabled) - self.assertNotIn("aws_mcp", disabled) @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches") @patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure") @@ -543,61 +491,16 @@ def test_agent_observability_respects_custom_disabled_instrumentations(self): self._configure_with_agent_observability() self.assertEqual(os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS), "custom_lib") - def test_base_otlp_endpoint_prevents_specific_endpoints_v1(self): + def test_base_otlp_endpoint_does_not_prevent_specific_endpoints(self): os.environ[OTEL_EXPORTER_OTLP_ENDPOINT] = "http://my-collector:4318" self._configure_with_agent_observability() - self.assertNotIn(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, os.environ) - self.assertNotIn(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, os.environ) - - def test_load_instrumentor_skip_behavior(self): - cases = [ - { - "env": {"AGENT_OBSERVABILITY_ENABLED": "true"}, - "entry_name": "openai_agents", - "dist_name": "opentelemetry-instrumentation-openai-agents-v2", - "should_load": False, - }, - { - "env": {"AGENT_OBSERVABILITY_ENABLED": "true"}, - "entry_name": "openai_agents", - "dist_name": "openinference-instrumentation-openai-agents", - "should_load": True, - }, - { - "env": {"AWS_AGENTIC_OBSERVABILITY_OPT_IN": "true"}, - "entry_name": "openai_agents", - "dist_name": "opentelemetry-instrumentation-openai-agents-v2", - "should_load": True, - }, - { - "env": {}, - "entry_name": "openai_agents", - "dist_name": "opentelemetry-instrumentation-openai-agents-v2", - "should_load": True, - }, - ] - for case in cases: - with self.subTest(case=case): - mock_entry_point = MagicMock() - mock_entry_point.name = case["entry_name"] - mock_entry_point.dist.name = case["dist_name"] - - distro = AwsOpenTelemetryDistro() - with patch.dict(os.environ, case["env"], clear=False): - with patch.object(OpenTelemetryDistro, "load_instrumentor") as mock_super: - distro.load_instrumentor(mock_entry_point) - if case["should_load"]: - mock_super.assert_called_once_with(mock_entry_point) - else: - mock_super.assert_not_called() + self.assertIn(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, os.environ) + self.assertIn(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, os.environ) def _configure_with_agent_observability(self, region="us-west-2"): with patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure"), patch( "amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches" ), patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_installed", return_value=False), patch( - "amazon.opentelemetry.distro.aws_opentelemetry_distro.is_aws_agentic_observability_opt_in", - return_value=False, - ), patch( "amazon.opentelemetry.distro.aws_opentelemetry_distro.is_agent_observability_enabled", return_value=True ), patch( "amazon.opentelemetry.distro.aws_opentelemetry_distro.get_aws_region", return_value=region diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_utils.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_utils.py index 9f2b0ce0c..4c0cd709f 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_utils.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_utils.py @@ -8,12 +8,9 @@ from amazon.opentelemetry.distro._utils import ( AGENT_OBSERVABILITY_ENABLED, - AWS_AGENTIC_OBSERVABILITY_OPT_IN, get_aws_region, get_aws_session, is_agent_observability_enabled, - is_agentic_observability_enabled, - is_aws_agentic_observability_opt_in, is_installed, ) @@ -108,41 +105,6 @@ def test_is_agent_observability_enabled_various_values(self): del os.environ[AGENT_OBSERVABILITY_ENABLED] self.assertFalse(is_agent_observability_enabled()) - def test_is_aws_agentic_observability_opt_in_various_values(self): - """Test is_aws_agentic_observability_opt_in with various environment variable values""" - os.environ[AWS_AGENTIC_OBSERVABILITY_OPT_IN] = "true" - self.assertTrue(is_aws_agentic_observability_opt_in()) - - os.environ[AWS_AGENTIC_OBSERVABILITY_OPT_IN] = "True" - self.assertTrue(is_aws_agentic_observability_opt_in()) - - os.environ[AWS_AGENTIC_OBSERVABILITY_OPT_IN] = "false" - self.assertFalse(is_aws_agentic_observability_opt_in()) - - if AWS_AGENTIC_OBSERVABILITY_OPT_IN in os.environ: - del os.environ[AWS_AGENTIC_OBSERVABILITY_OPT_IN] - self.assertFalse(is_aws_agentic_observability_opt_in()) - - def test_is_agentic_observability_enabled_combined(self): - """Test is_agentic_observability_enabled returns True if either env var is set""" - os.environ.pop(AGENT_OBSERVABILITY_ENABLED, None) - os.environ.pop(AWS_AGENTIC_OBSERVABILITY_OPT_IN, None) - self.assertFalse(is_agentic_observability_enabled()) - - os.environ[AGENT_OBSERVABILITY_ENABLED] = "true" - self.assertTrue(is_agentic_observability_enabled()) - os.environ.pop(AGENT_OBSERVABILITY_ENABLED, None) - - os.environ[AWS_AGENTIC_OBSERVABILITY_OPT_IN] = "true" - self.assertTrue(is_agentic_observability_enabled()) - os.environ.pop(AWS_AGENTIC_OBSERVABILITY_OPT_IN, None) - - os.environ[AGENT_OBSERVABILITY_ENABLED] = "true" - os.environ[AWS_AGENTIC_OBSERVABILITY_OPT_IN] = "true" - self.assertTrue(is_agentic_observability_enabled()) - os.environ.pop(AGENT_OBSERVABILITY_ENABLED, None) - os.environ.pop(AWS_AGENTIC_OBSERVABILITY_OPT_IN, None) - def test_get_aws_session_with_botocore(self): """Test get_aws_session when botocore is installed""" with patch("amazon.opentelemetry.distro._utils.IS_BOTOCORE_INSTALLED", True): From 821b24b556b33ec4e20ba107688e49d53610f890 Mon Sep 17 00:00:00 2001 From: wangzlei Date: Tue, 21 Apr 2026 16:42:07 -0700 Subject: [PATCH 2/2] feat: auto-detect mutual exclusion between AWS native and third-party agentic instrumentors When AGENT_OBSERVABILITY_ENABLED=true, auto-detect registered third-party instrumentors (e.g. OpenInference) and disable conflicting AWS native ones to prevent double instrumentation. Add AWS_AGENTIC_INSTRUMENTATION_OPT_IN env var to let users override auto-detection and force AWS native instrumentors over third-party ones. --- .gitignore | 1 + CHANGELOG.md | 3 + .../src/amazon/opentelemetry/distro/_utils.py | 1 + .../distro/aws_opentelemetry_distro.py | 51 ++++++++- .../aws/traces/test_otlp_aws_span_exporter.py | 28 ++--- .../distro/test_aws_opentelemetry_distro.py | 106 +++++++++++++++++- 6 files changed, 162 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 95031f272..bfad17d97 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ pip-log.txt # Unit test / coverage reports coverage.xml .coverage +.coverage.* .nox .tox .cache diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b718c802..01c18f101 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t ## Unreleased +- feat: auto-detect and mutually exclude AWS native vs third-party agentic instrumentors; add `AWS_AGENTIC_INSTRUMENTATION_OPT_IN` env var to override auto-detection + ([#729](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/729)) + ## v0.17.0 - 2026-04-08 - feat: support environment-configured endpoint visibility for HTTP operation names diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py index bd5f2a367..4ac8f7921 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py @@ -13,6 +13,7 @@ AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED" OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS = "OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS" + def is_installed(req: str) -> bool: """Is the given required package installed?""" req = Requirement(req) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py index bef3fdcd4..516e265fd 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py @@ -98,14 +98,20 @@ def _check_otel_version_compatibility(): OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, - OTEL_TRACES_SAMPLER, ) +from opentelemetry.util._importlib_metadata import EntryPoint, entry_points _logger: Logger = getLogger(__name__) # Suppress configurator warnings from auto-instrumentation _load._logger.setLevel(LEVELS.get(os.environ.get(OTEL_PYTHON_LOG_LEVEL, "error").lower(), ERROR)) +# When set to "true", opt in to AWS agentic instrumentors over third-party ones (e.g. OpenInference), +# even if both are installed. By default, auto-detection prefers third-party when present. +AWS_AGENTIC_INSTRUMENTATION_OPT_IN = "AWS_AGENTIC_INSTRUMENTATION_OPT_IN" + +# Maps third-party instrumentor entry point names to their AWS native equivalents. +# Used for mutual exclusion: only one side instruments each library at a time. _THIRDPARTY_TO_AWS_NATIVE = { "crewai": "aws_crewai", "langchain": "aws_langchain", @@ -115,6 +121,10 @@ def _check_otel_version_compatibility(): "openai_agents": "aws_openai_agents", } +# Dist names owned by ADOT that register entry points with the same names as third-party ones. +# These are excluded from third-party detection to avoid false positives. +_ADOT_OWNED_DISTS = {"opentelemetry-instrumentation-openai-agents-v2"} + class AwsOpenTelemetryDistro(OpenTelemetryDistro): def _configure(self, **kwargs): @@ -207,7 +217,6 @@ def _configure(self, **kwargs): "Please set AWS_REGION environment variable or configure OTLP endpoints manually." ) - os.environ.setdefault(OTEL_TRACES_SAMPLER, "parentbased_always_on") os.environ.setdefault( OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "http,sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector," @@ -219,9 +228,45 @@ def _configure(self, **kwargs): os.environ.setdefault(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "false") os.environ.setdefault("CREWAI_DISABLE_TELEMETRY", "true") - super(AwsOpenTelemetryDistro, self)._configure() if kwargs.get("apply_patches", True): apply_instrumentation_patches() + def load_instrumentor(self, entry_point: EntryPoint, **kwargs): + """Mutual exclusion between AWS native and third-party agentic instrumentors. + + When agent observability is enabled: + - Skip ADOT-owned dists that duplicate an aws_* entry point (e.g. openai-agents-v2). + - Auto-detect: if a third-party instrumentor is registered for the same library, + skip the AWS native one. If not, skip the third-party one. + - AWS_AGENTIC_INSTRUMENTATION_OPT_IN=true overrides auto-detection and forces + AWS native instrumentors, skipping third-party ones. + """ + if is_agent_observability_enabled() and self._should_skip_instrumentor(entry_point): + return + super().load_instrumentor(entry_point, **kwargs) + + @staticmethod + def _should_skip_instrumentor(entry_point): + if entry_point.dist and entry_point.dist.name in _ADOT_OWNED_DISTS: + return True + + prefer_native = os.environ.get(AWS_AGENTIC_INSTRUMENTATION_OPT_IN, "false").lower() == "true" + third_party_names = { + ep.name + for ep in entry_points(group="opentelemetry_instrumentor") + if not (ep.dist and ep.dist.name in _ADOT_OWNED_DISTS) + } + + if entry_point.name in _THIRDPARTY_TO_AWS_NATIVE.values() and not prefer_native: + for tp_name, aws_name in _THIRDPARTY_TO_AWS_NATIVE.items(): + if entry_point.name == aws_name and tp_name in third_party_names: + _logger.debug("Skipping %s: third-party %s is registered", aws_name, tp_name) + return True + + if entry_point.name in _THIRDPARTY_TO_AWS_NATIVE and prefer_native: + _logger.debug("Skipping third-party %s: AWS native preferred", entry_point.name) + return True + + return False diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py index 8f8631a9d..fb5052484 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py @@ -50,9 +50,7 @@ def test_aws_headers_applied(self): self.assertEqual(exporter._session.headers["X-Custom-Header"], "custom-value") self.assertIn("User-Agent", exporter._session.headers) - @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" - ) + @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled") def test_ensure_llo_handler_when_disabled(self, mock_is_enabled): # Test _ensure_llo_handler when agent observability is disabled mock_is_enabled.return_value = False @@ -66,9 +64,7 @@ def test_ensure_llo_handler_when_disabled(self, mock_is_enabled): mock_is_enabled.assert_called_once() @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") - @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" - ) + @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled") @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler") def test_ensure_llo_handler_lazy_initialization( self, mock_llo_handler_class, mock_is_enabled, mock_get_logger_provider @@ -102,9 +98,7 @@ def test_ensure_llo_handler_lazy_initialization( mock_get_logger_provider.assert_not_called() @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") - @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" - ) + @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled") def test_ensure_llo_handler_with_existing_logger_provider(self, mock_is_enabled, mock_get_logger_provider): # Test when logger_provider is already provided mock_is_enabled.return_value = True @@ -129,9 +123,7 @@ def test_ensure_llo_handler_with_existing_logger_provider(self, mock_is_enabled, mock_get_logger_provider.assert_not_called() @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") - @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" - ) + @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled") def test_ensure_llo_handler_get_logger_provider_fails(self, mock_is_enabled, mock_get_logger_provider): # Test when get_logger_provider raises exception mock_is_enabled.return_value = True @@ -145,9 +137,7 @@ def test_ensure_llo_handler_get_logger_provider_fails(self, mock_is_enabled, moc self.assertFalse(result) self.assertIsNone(exporter._llo_handler) - @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" - ) + @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled") def test_export_with_llo_disabled(self, mock_is_enabled): # Test export when LLO is disabled mock_is_enabled.return_value = False @@ -166,9 +156,7 @@ def test_export_with_llo_disabled(self, mock_is_enabled): mock_parent_export.assert_called_once_with(spans) self.assertIsNone(exporter._llo_handler) - @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" - ) + @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled") @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler") def test_export_with_llo_enabled(self, mock_llo_handler_class, mock_get_logger_provider, mock_is_enabled): @@ -198,9 +186,7 @@ def test_export_with_llo_enabled(self, mock_llo_handler_class, mock_get_logger_p mock_llo_handler.process_spans.assert_called_once_with(original_spans) mock_parent_export.assert_called_once_with(processed_spans) - @patch( - "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled" - ) + @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled") @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider") @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler") def test_export_with_llo_processing_failure( diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelemetry_distro.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelemetry_distro.py index 5f80b750c..f1e929d7c 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelemetry_distro.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelemetry_distro.py @@ -9,9 +9,7 @@ from unittest.mock import MagicMock, patch from amazon.opentelemetry.distro.aws_opentelemetry_configurator import APPLICATION_SIGNALS_ENABLED_CONFIG -from amazon.opentelemetry.distro.aws_opentelemetry_distro import ( - AwsOpenTelemetryDistro, -) +from amazon.opentelemetry.distro.aws_opentelemetry_distro import AwsOpenTelemetryDistro from opentelemetry import propagate from opentelemetry.distro import OpenTelemetryDistro from opentelemetry.environment_variables import ( @@ -59,6 +57,8 @@ def setUp(self): "DJANGO_SETTINGS_MODULE", OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, + "AWS_AGENTIC_INSTRUMENTATION_OPT_IN", + "CREWAI_DISABLE_TELEMETRY", ] # First, save all current values @@ -489,7 +489,8 @@ def test_application_signals_dimensions_disabled_with_agent_observability( def test_agent_observability_respects_custom_disabled_instrumentations(self): os.environ[OTEL_PYTHON_DISABLED_INSTRUMENTATIONS] = "custom_lib" self._configure_with_agent_observability() - self.assertEqual(os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS), "custom_lib") + disabled = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "") + self.assertTrue(disabled.startswith("custom_lib")) def test_base_otlp_endpoint_does_not_prevent_specific_endpoints(self): os.environ[OTEL_EXPORTER_OTLP_ENDPOINT] = "http://my-collector:4318" @@ -497,6 +498,103 @@ def test_base_otlp_endpoint_does_not_prevent_specific_endpoints(self): self.assertIn(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, os.environ) self.assertIn(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, os.environ) + @staticmethod + def _make_ep(name, dist_name=None): + ep = MagicMock() + ep.name = name + if dist_name: + ep.dist = MagicMock() + ep.dist.name = dist_name + else: + ep.dist = None + return ep + + def _load_instrumentor_with_agent(self, ep, third_party_eps=None, prefer_native=False): + """Helper to test load_instrumentor with agent observability enabled.""" + distro = AwsOpenTelemetryDistro() + AwsOpenTelemetryDistro._third_party_instrumentors = None + if prefer_native: + os.environ["AWS_AGENTIC_INSTRUMENTATION_OPT_IN"] = "true" + with patch( + "amazon.opentelemetry.distro.aws_opentelemetry_distro.is_agent_observability_enabled", return_value=True + ), patch( + "amazon.opentelemetry.distro.aws_opentelemetry_distro.entry_points", return_value=third_party_eps or [] + ), patch.object( + OpenTelemetryDistro, "load_instrumentor" + ) as mock_super: + distro.load_instrumentor(ep) + return mock_super + + def test_skip_adot_owned_dist(self): + """ADOT-owned v2 package should be skipped when agent observability is enabled.""" + ep = self._make_ep("openai_agents", "opentelemetry-instrumentation-openai-agents-v2") + mock_super = self._load_instrumentor_with_agent(ep) + mock_super.assert_not_called() + + def test_load_adot_owned_dist_when_agent_disabled(self): + """ADOT-owned v2 package should load when agent observability is disabled.""" + distro = AwsOpenTelemetryDistro() + ep = self._make_ep("openai_agents", "opentelemetry-instrumentation-openai-agents-v2") + with patch( + "amazon.opentelemetry.distro.aws_opentelemetry_distro.is_agent_observability_enabled", return_value=False + ), patch.object(OpenTelemetryDistro, "load_instrumentor") as mock_super: + distro.load_instrumentor(ep) + mock_super.assert_called_once_with(ep) + + def test_skip_native_when_third_party_registered(self): + """aws_langchain should be skipped when OpenInference langchain is registered.""" + ep = self._make_ep("aws_langchain", "aws-opentelemetry-distro") + third_party = [self._make_ep("langchain", "openinference-instrumentation-langchain")] + mock_super = self._load_instrumentor_with_agent(ep, third_party_eps=third_party) + mock_super.assert_not_called() + + def test_load_native_when_no_third_party(self): + """aws_langchain should load when no third-party langchain is registered.""" + ep = self._make_ep("aws_langchain", "aws-opentelemetry-distro") + mock_super = self._load_instrumentor_with_agent(ep, third_party_eps=[]) + mock_super.assert_called_once_with(ep) + + def test_skip_third_party_when_prefer_native(self): + """Third-party langchain should be skipped when AWS_AGENTIC_INSTRUMENTATION_OPT_IN=true.""" + ep = self._make_ep("langchain", "openinference-instrumentation-langchain") + third_party = [self._make_ep("langchain", "openinference-instrumentation-langchain")] + mock_super = self._load_instrumentor_with_agent(ep, third_party_eps=third_party, prefer_native=True) + mock_super.assert_not_called() + + def test_load_third_party_when_not_prefer_native(self): + """Third-party langchain should load when auto-detect (default).""" + ep = self._make_ep("langchain", "openinference-instrumentation-langchain") + third_party = [self._make_ep("langchain", "openinference-instrumentation-langchain")] + mock_super = self._load_instrumentor_with_agent(ep, third_party_eps=third_party) + mock_super.assert_called_once_with(ep) + + def test_load_native_when_prefer_native(self): + """aws_langchain should load when AWS_AGENTIC_INSTRUMENTATION_OPT_IN=true even if third-party registered.""" + ep = self._make_ep("aws_langchain", "aws-opentelemetry-distro") + third_party = [self._make_ep("langchain", "openinference-instrumentation-langchain")] + mock_super = self._load_instrumentor_with_agent(ep, third_party_eps=third_party, prefer_native=True) + mock_super.assert_called_once_with(ep) + + def test_load_regular_instrumentor(self): + """Regular instrumentors should always be loaded.""" + ep = self._make_ep("flask", "opentelemetry-instrumentation-flask") + mock_super = self._load_instrumentor_with_agent(ep) + mock_super.assert_called_once_with(ep) + + def test_openinference_openai_agents_skips_native(self): + """When OpenInference openai_agents is registered, aws_openai_agents should be skipped.""" + ep = self._make_ep("aws_openai_agents", "aws-opentelemetry-distro") + third_party = [self._make_ep("openai_agents", "openinference-instrumentation-openai-agents")] + mock_super = self._load_instrumentor_with_agent(ep, third_party_eps=third_party) + mock_super.assert_not_called() + + def test_adot_v2_openai_agents_not_treated_as_third_party(self): + """ADOT v2 openai_agents should not cause aws_openai_agents to be skipped.""" + ep = self._make_ep("aws_openai_agents", "aws-opentelemetry-distro") + third_party = [self._make_ep("openai_agents", "opentelemetry-instrumentation-openai-agents-v2")] + mock_super = self._load_instrumentor_with_agent(ep, third_party_eps=third_party) + mock_super.assert_called_once_with(ep) + def _configure_with_agent_observability(self, region="us-west-2"): with patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure"), patch( "amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches"