feat: [BREAKING CHANGE] introduce AWS_AGENTIC_OBSERVABILITY_OPT_IN and refactor agent observability config (#691)

*Description of changes:*

- add note for `AGENT_OBSERVABILITY_VERSION` with a new env var
`AWS_AGENTIC_OBSERVABILITY_OPT_IN` that controls ADOT-native agentic
instrumentation. When enabled, ADOT disables third-party agentic
instrumentations.

- `AGENT_OBSERVABILITY_ENABLED` is maintained for backwards
compatibility, disabling native ADOT instrumentations.

- **BREAKING CHANGE**: enable ALL HTTP library instrumentations for
`AGENT_OBSERVABILITY_ENABLED` and `AWS_AGENTIC_OBSERVABILITY_OPT_IN`
path

Other changes:
- Add `OTEL_PYTHON_LOG_CORRELATION` to common agent observability config
- Remove redundant `OTEL_TRACES_SAMPLER` env variable
- Remove hardcoded `localhost` OTLP endpoints from opt-in path


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: liustve

CHANGELOG.md

@@ -12,6 +12,8 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t
 
 ## Unreleased
 
+- feat: [BREAKING CHANGE] introduce AWS_AGENTIC_OBSERVABILITY_OPT_IN and refactor agent observability config
+  ([#691](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/691))
 - feat: propagate HTTP context for MCP requests and prefix all span names with mcp
   ([#683](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/683))
 - feat: add threading instrumentation dependency

aws-opentelemetry-distro/pyproject.toml

@@ -108,6 +108,10 @@ aws_distro = "amazon.opentelemetry.distro.aws_opentelemetry_distro:AwsOpenTeleme
 aws_crewai = "amazon.opentelemetry.distro.instrumentation.crewai:CrewAIInstrumentor"
 aws_langchain = "amazon.opentelemetry.distro.instrumentation.langchain:LangChainInstrumentor"
 aws_mcp = "amazon.opentelemetry.distro.instrumentation.mcp:McpInstrumentor"
+# Wraps the upstream OTel OpenAI Agents instrumentor under a unique entry point name.
+# This allows us to disable 3rd party "openai_agents" entry points in the opt-in path
+# while keeping OpenAI Agents instrumentation active via this alias.
+aws_openai_agents = "opentelemetry.instrumentation.openai_agents:OpenAIAgentsInstrumentor"
 
 [project.urls]
 Homepage = "https://github.com/aws-observability/aws-otel-python-instrumentation/tree/main/aws-opentelemetry-distro"

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py

@@ -10,7 +10,9 @@
 
 _logger: Logger = getLogger(__name__)
 
+# Maintained for backwards compatibility. New users should use AWS_AGENTIC_OBSERVABILITY_OPT_IN instead.
 AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED"
+AWS_AGENTIC_OBSERVABILITY_OPT_IN = "AWS_AGENTIC_OBSERVABILITY_OPT_IN"
 OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS = "OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS"
 
 
@@ -35,10 +37,20 @@ def is_installed(req: str) -> bool:
 
 
 def is_agent_observability_enabled() -> bool:
-    """Is the Agentic AI monitoring flag set to true?"""
+    # Maintained for backwards compatibility. New users should use AWS_AGENTIC_OBSERVABILITY_OPT_IN instead.
     return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true"
 
 
+def is_aws_agentic_observability_opt_in() -> bool:
+    """Is the AI observability opt-in flag set to true?"""
+    return os.environ.get(AWS_AGENTIC_OBSERVABILITY_OPT_IN, "false").lower() == "true"
+
+
+def is_agentic_observability_enabled() -> bool:
+    """Returns True if either AGENT_OBSERVABILITY_ENABLED or AWS_AGENTIC_OBSERVABILITY_OPT_IN is set to true."""
+    return is_agent_observability_enabled() or is_aws_agentic_observability_opt_in()
+
+
 def should_add_application_signals_dimensions() -> bool:
     """Should Service and Environment Application Signals dimensions be added to EMF logs?"""
     return os.environ.get(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "true").lower() == "true"

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -17,7 +17,7 @@
 
 from amazon.opentelemetry.distro._aws_attribute_keys import AWS_LOCAL_SERVICE, AWS_SERVICE_TYPE
 from amazon.opentelemetry.distro._aws_resource_attribute_configurator import get_service_attribute
-from amazon.opentelemetry.distro._utils import get_aws_session, is_agent_observability_enabled
+from amazon.opentelemetry.distro._utils import get_aws_session, is_agentic_observability_enabled
 from amazon.opentelemetry.distro.always_record_sampler import AlwaysRecordSampler
 from amazon.opentelemetry.distro.attribute_propagating_span_processor_builder import (
     AttributePropagatingSpanProcessorBuilder,
@@ -204,7 +204,7 @@ def _initialize_components():
             AwsEksResourceDetector(),
             AwsEcsResourceDetector(),
         ]
-        if not (_is_lambda_environment() or is_agent_observability_enabled())
+        if not (_is_lambda_environment() or is_agentic_observability_enabled())
         else []
     )
 
@@ -326,7 +326,7 @@ def _export_unsampled_span_for_lambda(trace_provider: TracerProvider, resource:
 
 
 def _export_unsampled_span_for_agent_observability(trace_provider: TracerProvider, resource: Resource = None):
-    if not is_agent_observability_enabled():
+    if not is_agentic_observability_enabled():
         return
 
     traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT)
@@ -467,7 +467,7 @@ def _customize_log_record_processor(logger_provider: LoggerProvider, log_exporte
     if not log_exporter:
         return
 
-    if is_agent_observability_enabled():
+    if is_agentic_observability_enabled():
         # pylint: disable=import-outside-toplevel
         from amazon.opentelemetry.distro.exporter.otlp.aws.logs._aws_cw_otlp_batch_log_record_processor import (
             AwsCloudWatchOtlpBatchLogRecordProcessor,
@@ -526,7 +526,7 @@ def _customize_span_processors(provider: TracerProvider, resource: Resource, sam
     # AI applications typically have low throughput traffic patterns and require
     # comprehensive monitoring to catch subtle failure modes like hallucinations
     # and quality degradation that sampling could miss.
-    if is_agent_observability_enabled():
+    if is_agentic_observability_enabled():
         _export_unsampled_span_for_agent_observability(provider, resource)
         baggage_keys.add("session.id")
 
@@ -633,7 +633,7 @@ def _customize_resource(resource: Resource) -> Resource:
 
     custom_attributes = {AWS_LOCAL_SERVICE: service_name}
 
-    if is_agent_observability_enabled():
+    if is_agentic_observability_enabled():
         # Add aws.service.type if it doesn't exist in the resource
         if resource and resource.attributes.get(AWS_SERVICE_TYPE) is None:
             # Set a default agent type for AI agent observability
@@ -919,7 +919,7 @@ def _create_aws_otlp_exporter(endpoint: str, service: str, region: str):
         from amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter import OTLPAwsSpanExporter
 
         if service == XRAY_SERVICE:
-            if is_agent_observability_enabled():
+            if is_agentic_observability_enabled():
                 # Span exporter needs an instance of logger provider in ai agent
                 # observability case because we need to split input/output prompts
                 # from span attributes and send them to the logs pipeline per

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py

@@ -20,6 +20,7 @@
     OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS,
     get_aws_region,
     is_agent_observability_enabled,
+    is_aws_agentic_observability_opt_in,
     is_installed,
 )
 from amazon.opentelemetry.distro.aws_opentelemetry_configurator import APPLICATION_SIGNALS_ENABLED_CONFIG
@@ -36,28 +37,35 @@
 from opentelemetry.instrumentation.auto_instrumentation import _load
 from opentelemetry.instrumentation.environment_variables import OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
 from opentelemetry.instrumentation.logging import LEVELS
-from opentelemetry.instrumentation.logging.environment_variables import OTEL_PYTHON_LOG_LEVEL
+from opentelemetry.instrumentation.logging.environment_variables import (
+    OTEL_PYTHON_LOG_CORRELATION,
+    OTEL_PYTHON_LOG_LEVEL,
+)
 from opentelemetry.sdk.environment_variables import (
     _OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED as OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED,
 )
 from opentelemetry.sdk.environment_variables import (
     OTEL_EXPORTER_OTLP_ENDPOINT,
     OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
     OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION,
-    OTEL_EXPORTER_OTLP_METRICS_ENDPOINT,
     OTEL_EXPORTER_OTLP_PROTOCOL,
     OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
-    OTEL_TRACES_SAMPLER,
 )
+from opentelemetry.util._importlib_metadata import EntryPoint
 
 _logger: Logger = getLogger(__name__)
 # Suppress configurator warnings from auto-instrumentation
 _load._logger.setLevel(LEVELS.get(os.environ.get(OTEL_PYTHON_LOG_LEVEL, "error").lower(), ERROR))
 
 
 AGENT_OBSERVABILITY_DISABLED_INSTRUMENTATIONS = (
-    "http,sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector,"
-    "urllib3,requests,system_metrics,google-genai,crewai,langchain,mcp"
+    "sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector,"
+    "system_metrics,google-genai,aws_crewai,aws_langchain,aws_mcp,aws_openai_agents"
+)
+
+AWS_AGENTIC_OBSERVABILITY_DISABLED_INSTRUMENTATIONS = (
+    "sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector,"
+    "system_metrics,google-genai,crewai,langchain,mcp,openai_agents"
 )
 
 
@@ -134,51 +142,66 @@ def _configure(self, **kwargs):
             OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, "base2_exponential_bucket_histogram"
         )
 
-        if is_agent_observability_enabled():
-            os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true")
-            os.environ.setdefault(OTEL_TRACES_SAMPLER, "parentbased_always_on")
-            os.environ.setdefault(
-                OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
-                AGENT_OBSERVABILITY_DISABLED_INSTRUMENTATIONS,
+        if is_aws_agentic_observability_opt_in():
+            _logger.info("AWS Agentic Observability enabled.")
+            self._configure_common_agent_observability(AWS_AGENTIC_OBSERVABILITY_DISABLED_INSTRUMENTATIONS)
+            os.environ.setdefault(OTEL_METRICS_EXPORTER, "otlp")
+            os.environ.setdefault("CREWAI_DISABLE_TELEMETRY", "true")
+        elif is_agent_observability_enabled():
+            # Maintained for backwards compatibility. New users should use AWS_AGENTIC_OBSERVABILITY_OPT_IN instead.
+            _logger.info(
+                "AGENT_OBSERVABILITY_ENABLED is set. Consider using AWS_AGENTIC_OBSERVABILITY_OPT_IN for ADOT Agentic Observability."
             )
-            os.environ.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true")
-            os.environ.setdefault(APPLICATION_SIGNALS_ENABLED_CONFIG, "false")
-            os.environ.setdefault(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "false")
-
-            version = os.environ.get("AGENT_OBSERVABILITY_VERSION", "1")
-            if version == "1":
-                _configure_agent_observability_v1(get_aws_region())
-            else:
-                _configure_agent_observability_v2()
+            self._configure_common_agent_observability(AGENT_OBSERVABILITY_DISABLED_INSTRUMENTATIONS)
+            os.environ.setdefault(OTEL_METRICS_EXPORTER, "awsemf")
+            region = get_aws_region()
+            if not os.environ.get(OTEL_EXPORTER_OTLP_ENDPOINT):
+                if region:
+                    os.environ.setdefault(
+                        OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, f"https://xray.{region}.amazonaws.com/v1/traces"
+                    )
+                    os.environ.setdefault(
+                        OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, f"https://logs.{region}.amazonaws.com/v1/logs"
+                    )
+                else:
+                    _logger.warning(
+                        "AWS region could not be determined. OTLP endpoints will not be automatically configured. "
+                        "Please set AWS_REGION environment variable or configure OTLP endpoints manually."
+                    )
 
         super(AwsOpenTelemetryDistro, self)._configure()
 
         if kwargs.get("apply_patches", True):
             apply_instrumentation_patches()
 
-
-def _configure_agent_observability_v1(region: str) -> None:
-    # "otlp" is already native OTel default, but we set them here to be explicit
-    # about intended configuration for agent observability
-    os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp")
-    os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp")
-    os.environ.setdefault(OTEL_METRICS_EXPORTER, "awsemf")
-    if not os.environ.get(OTEL_EXPORTER_OTLP_ENDPOINT):
-        if region:
-            os.environ.setdefault(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, f"https://xray.{region}.amazonaws.com/v1/traces")
-            os.environ.setdefault(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, f"https://logs.{region}.amazonaws.com/v1/logs")
-        else:
-            _logger.warning(
-                "AWS region could not be determined. OTLP endpoints will not be automatically configured. "
-                "Please set AWS_REGION environment variable or configure OTLP endpoints manually."
-            )
-
-
-def _configure_agent_observability_v2() -> None:
-    os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp")
-    os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp")
-    os.environ.setdefault(OTEL_METRICS_EXPORTER, "otlp")
-    if not os.environ.get(OTEL_EXPORTER_OTLP_ENDPOINT):
-        os.environ.setdefault(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, "http://localhost:4318/v1/traces")
-        os.environ.setdefault(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, "http://localhost:4318/v1/logs")
-        os.environ.setdefault(OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, "http://localhost:4318/v1/metrics")
+    def load_instrumentor(self, entry_point: EntryPoint, **kwargs):
+        if self._should_skip_instrumentor(entry_point):
+            return
+        super().load_instrumentor(entry_point, **kwargs)
+
+    @staticmethod
+    def _should_skip_instrumentor(entry_point: EntryPoint) -> bool:
+        # Some third-party SDKs register the same entry point name as the upstream
+        # OTel packages that we depend on. For Agentic Observability legacy mode, skip our bundled
+        # OTel instrumentation so that existing third-party setups are not brokens.
+        if (
+            is_agent_observability_enabled()
+            and not is_aws_agentic_observability_opt_in()
+            and entry_point.dist
+            and entry_point.name == "openai_agents"
+            and entry_point.dist.name == "opentelemetry-instrumentation-openai-agents-v2"
+        ):
+            return True
+        # TODO: add additional skip conditions here as needed
+        return False
+
+    @staticmethod
+    def _configure_common_agent_observability(disabled_instrumentations: str) -> None:
+        os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true")
+        os.environ.setdefault(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, disabled_instrumentations)
+        os.environ.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true")
+        os.environ.setdefault(OTEL_PYTHON_LOG_CORRELATION, "true")
+        os.environ.setdefault(APPLICATION_SIGNALS_ENABLED_CONFIG, "false")
+        os.environ.setdefault(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "false")
+        os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp")
+        os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp")

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/common/_aws_http_headers.py

@@ -1,14 +1,14 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
+from amazon.opentelemetry.distro._utils import is_agentic_observability_enabled
 from amazon.opentelemetry.distro.version import __version__
 
 
 def build_user_agent() -> str:
     user_agent = f"ADOT-OTLP-Exporter-Python/{__version__}"
 
-    if is_agent_observability_enabled():
+    if is_agentic_observability_enabled():
         user_agent = f"ADOT-GenAI-OTLP-Exporter-Python/{__version__}"
     return user_agent
 

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py

@@ -6,7 +6,7 @@
 
 from botocore.session import Session
 
-from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
+from amazon.opentelemetry.distro._utils import is_agentic_observability_enabled
 from amazon.opentelemetry.distro.exporter.otlp.aws.common._aws_http_headers import _OTLP_AWS_HTTP_HEADERS
 from amazon.opentelemetry.distro.exporter.otlp.aws.common.aws_auth_session import AwsAuthSession
 from amazon.opentelemetry.distro.llo_handler import LLOHandler
@@ -61,7 +61,7 @@ def __init__(
 
     def _ensure_llo_handler(self):
         """Lazily initialize LLO handler when needed to avoid initialization order issues"""
-        if self._llo_handler is None and is_agent_observability_enabled():
+        if self._llo_handler is None and is_agentic_observability_enabled():
             if self._logger_provider is None:
                 try:
                     self._logger_provider = get_logger_provider()
@@ -77,7 +77,7 @@ def _ensure_llo_handler(self):
 
     def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
         try:
-            if is_agent_observability_enabled() and self._ensure_llo_handler():
+            if is_agentic_observability_enabled() and self._ensure_llo_handler():
                 llo_processed_spans = self._llo_handler.process_spans(spans)
                 return super().export(llo_processed_spans)
         except Exception:  # pylint: disable=broad-exception-caught

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_starlette_patches.py

@@ -3,7 +3,7 @@
 # Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
 from logging import Logger, getLogger
 
-from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
+from amazon.opentelemetry.distro._utils import is_agentic_observability_enabled
 
 _logger: Logger = getLogger(__name__)
 
@@ -24,7 +24,7 @@ def _apply_starlette_instrumentation_patches() -> None:
         #
         # Issue for tracking a feature to customize this setting within Starlette:
         # https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3725
-        if is_agent_observability_enabled():
+        if is_agentic_observability_enabled():
             original_init = OpenTelemetryMiddleware.__init__
 
             def patched_init(self, app, **kwargs):