Skip to content

Commit 84ea188

Browse files
wangzleiwangzlei
committed
feat: auto-detect mutual exclusion between AWS native and third-party agentic instrumentors
When AGENT_OBSERVABILITY_ENABLED=true, auto-detect registered third-party instrumentors (e.g. OpenInference) and disable conflicting AWS native ones to prevent double instrumentation. Add AWS_AGENTIC_INSTRUMENTATION_OPT_IN env var to let users override auto-detection and force AWS native instrumentors over third-party ones.
1 parent 537ac16 commit 84ea188

6 files changed

Lines changed: 147 additions & 28 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ pip-log.txt
3030
# Unit test / coverage reports
3131
coverage.xml
3232
.coverage
33+
.coverage.*
3334
.nox
3435
.tox
3536
.cache

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t
1212

1313
## Unreleased
1414

15+
- feat: auto-detect and mutually exclude AWS native vs third-party agentic instrumentors; add `AWS_AGENTIC_INSTRUMENTATION_OPT_IN` env var to override auto-detection
16+
([#729](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/729))
1517
- feat: support environment-configured endpoint visibility for HTTP operation names
1618
([#718](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/718))
1719
- fix(lambda-layer): Disable all agentic instrumentation in Lambda by default

‎aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED"
1414
OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS = "OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS"
1515

16+
1617
def is_installed(req: str) -> bool:
1718
"""Is the given required package installed?"""
1819
req = Requirement(req)

‎aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,19 @@ def _check_otel_version_compatibility():
100100
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
101101
OTEL_TRACES_SAMPLER,
102102
)
103+
from opentelemetry.util._importlib_metadata import entry_points
103104

104105
_logger: Logger = getLogger(__name__)
105106
# Suppress configurator warnings from auto-instrumentation
106107
_load._logger.setLevel(LEVELS.get(os.environ.get(OTEL_PYTHON_LOG_LEVEL, "error").lower(), ERROR))
107108

108109

110+
# When set to "true", opt in to AWS agentic instrumentors over third-party ones (e.g. OpenInference),
111+
# even if both are installed. By default, auto-detection prefers third-party when present.
112+
AWS_AGENTIC_INSTRUMENTATION_OPT_IN = "AWS_AGENTIC_INSTRUMENTATION_OPT_IN"
113+
114+
# Maps third-party instrumentor entry point names to their AWS native equivalents.
115+
# Used for mutual exclusion: only one side instruments each library at a time.
109116
_THIRDPARTY_TO_AWS_NATIVE = {
110117
"crewai": "aws_crewai",
111118
"langchain": "aws_langchain",
@@ -208,20 +215,41 @@ def _configure(self, **kwargs):
208215
)
209216

210217
os.environ.setdefault(OTEL_TRACES_SAMPLER, "parentbased_always_on")
218+
219+
# Mutual exclusion between AWS native and third-party agentic instrumentors.
220+
# AWS_AGENTIC_INSTRUMENTATION_OPT_IN=true overrides auto-detection
221+
# and forces AWS native instrumentors, disabling third-party ones.
222+
# Default (auto-detect): if a third-party instrumentor is registered, disable the AWS native one.
223+
prefer_native = os.environ.get(AWS_AGENTIC_INSTRUMENTATION_OPT_IN, "false").lower() == "true"
224+
registered = {ep.name for ep in entry_points(group="opentelemetry_instrumentor")}
225+
disable_native = []
226+
disable_third_party = []
227+
for third_party, aws_native in _THIRDPARTY_TO_AWS_NATIVE.items():
228+
if third_party in registered:
229+
if prefer_native:
230+
disable_third_party.append(third_party)
231+
else:
232+
disable_native.append(aws_native)
233+
else:
234+
disable_third_party.append(third_party)
235+
211236
os.environ.setdefault(
212237
OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
213238
"http,sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector,"
214239
"urllib3,requests,system_metrics,google-genai,jinja2",
215240
)
241+
# Force-disable conflicting instrumentors to prevent double instrumentation.
242+
to_disable = disable_native if not prefer_native else disable_third_party
243+
if to_disable:
244+
existing = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "")
245+
os.environ[OTEL_PYTHON_DISABLED_INSTRUMENTATIONS] = existing + "," + ",".join(dict.fromkeys(to_disable))
216246
os.environ.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true")
217247
os.environ.setdefault(OTEL_PYTHON_LOG_CORRELATION, "true")
218248
os.environ.setdefault(APPLICATION_SIGNALS_ENABLED_CONFIG, "false")
219249
os.environ.setdefault(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "false")
220250
os.environ.setdefault("CREWAI_DISABLE_TELEMETRY", "true")
221251

222-
223252
super(AwsOpenTelemetryDistro, self)._configure()
224253

225254
if kwargs.get("apply_patches", True):
226255
apply_instrumentation_patches()
227-

‎aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,7 @@ def test_aws_headers_applied(self):
5050
self.assertEqual(exporter._session.headers["X-Custom-Header"], "custom-value")
5151
self.assertIn("User-Agent", exporter._session.headers)
5252

53-
@patch(
54-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
55-
)
53+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
5654
def test_ensure_llo_handler_when_disabled(self, mock_is_enabled):
5755
# Test _ensure_llo_handler when agent observability is disabled
5856
mock_is_enabled.return_value = False
@@ -66,9 +64,7 @@ def test_ensure_llo_handler_when_disabled(self, mock_is_enabled):
6664
mock_is_enabled.assert_called_once()
6765

6866
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
69-
@patch(
70-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
71-
)
67+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
7268
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
7369
def test_ensure_llo_handler_lazy_initialization(
7470
self, mock_llo_handler_class, mock_is_enabled, mock_get_logger_provider
@@ -102,9 +98,7 @@ def test_ensure_llo_handler_lazy_initialization(
10298
mock_get_logger_provider.assert_not_called()
10399

104100
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
105-
@patch(
106-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
107-
)
101+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
108102
def test_ensure_llo_handler_with_existing_logger_provider(self, mock_is_enabled, mock_get_logger_provider):
109103
# Test when logger_provider is already provided
110104
mock_is_enabled.return_value = True
@@ -129,9 +123,7 @@ def test_ensure_llo_handler_with_existing_logger_provider(self, mock_is_enabled,
129123
mock_get_logger_provider.assert_not_called()
130124

131125
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
132-
@patch(
133-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
134-
)
126+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
135127
def test_ensure_llo_handler_get_logger_provider_fails(self, mock_is_enabled, mock_get_logger_provider):
136128
# Test when get_logger_provider raises exception
137129
mock_is_enabled.return_value = True
@@ -145,9 +137,7 @@ def test_ensure_llo_handler_get_logger_provider_fails(self, mock_is_enabled, moc
145137
self.assertFalse(result)
146138
self.assertIsNone(exporter._llo_handler)
147139

148-
@patch(
149-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
150-
)
140+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
151141
def test_export_with_llo_disabled(self, mock_is_enabled):
152142
# Test export when LLO is disabled
153143
mock_is_enabled.return_value = False
@@ -166,9 +156,7 @@ def test_export_with_llo_disabled(self, mock_is_enabled):
166156
mock_parent_export.assert_called_once_with(spans)
167157
self.assertIsNone(exporter._llo_handler)
168158

169-
@patch(
170-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
171-
)
159+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
172160
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
173161
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
174162
def test_export_with_llo_enabled(self, mock_llo_handler_class, mock_get_logger_provider, mock_is_enabled):
@@ -198,9 +186,7 @@ def test_export_with_llo_enabled(self, mock_llo_handler_class, mock_get_logger_p
198186
mock_llo_handler.process_spans.assert_called_once_with(original_spans)
199187
mock_parent_export.assert_called_once_with(processed_spans)
200188

201-
@patch(
202-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
203-
)
189+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
204190
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
205191
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
206192
def test_export_with_llo_processing_failure(

‎aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelemetry_distro.py‎

Lines changed: 106 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,8 @@
99
from unittest.mock import MagicMock, patch
1010

1111
from amazon.opentelemetry.distro.aws_opentelemetry_configurator import APPLICATION_SIGNALS_ENABLED_CONFIG
12-
from amazon.opentelemetry.distro.aws_opentelemetry_distro import (
13-
AwsOpenTelemetryDistro,
14-
)
12+
from amazon.opentelemetry.distro.aws_opentelemetry_distro import AwsOpenTelemetryDistro
1513
from opentelemetry import propagate
16-
from opentelemetry.distro import OpenTelemetryDistro
1714
from opentelemetry.environment_variables import (
1815
OTEL_LOGS_EXPORTER,
1916
OTEL_METRICS_EXPORTER,
@@ -59,6 +56,8 @@ def setUp(self):
5956
"DJANGO_SETTINGS_MODULE",
6057
OTEL_EXPORTER_OTLP_ENDPOINT,
6158
OTEL_EXPORTER_OTLP_METRICS_ENDPOINT,
59+
"AWS_AGENTIC_INSTRUMENTATION_OPT_IN",
60+
"CREWAI_DISABLE_TELEMETRY",
6261
]
6362

6463
# First, save all current values
@@ -489,14 +488,116 @@ def test_application_signals_dimensions_disabled_with_agent_observability(
489488
def test_agent_observability_respects_custom_disabled_instrumentations(self):
490489
os.environ[OTEL_PYTHON_DISABLED_INSTRUMENTATIONS] = "custom_lib"
491490
self._configure_with_agent_observability()
492-
self.assertEqual(os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS), "custom_lib")
491+
disabled = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "")
492+
self.assertTrue(disabled.startswith("custom_lib"))
493493

494494
def test_base_otlp_endpoint_does_not_prevent_specific_endpoints(self):
495495
os.environ[OTEL_EXPORTER_OTLP_ENDPOINT] = "http://my-collector:4318"
496496
self._configure_with_agent_observability()
497497
self.assertIn(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, os.environ)
498498
self.assertIn(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, os.environ)
499499

500+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.entry_points")
501+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.get_aws_region")
502+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_agent_observability_enabled")
503+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_installed")
504+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches")
505+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure")
506+
def test_auto_detect_disables_native_when_thirdparty_registered(
507+
self, mock_super, mock_patches, mock_installed, mock_agent, mock_region, mock_eps
508+
):
509+
"""When a third-party instrumentor is registered, the corresponding AWS native one should be disabled."""
510+
mock_agent.return_value = True
511+
mock_region.return_value = "us-east-1"
512+
mock_installed.return_value = False
513+
514+
mock_langchain_ep = MagicMock()
515+
mock_langchain_ep.name = "langchain"
516+
mock_crewai_ep = MagicMock()
517+
mock_crewai_ep.name = "crewai"
518+
mock_eps.return_value = [mock_langchain_ep, mock_crewai_ep]
519+
520+
AwsOpenTelemetryDistro()._configure()
521+
522+
disabled = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "")
523+
self.assertIn("aws_langchain", disabled)
524+
self.assertIn("aws_crewai", disabled)
525+
self.assertNotIn("aws_mcp", disabled)
526+
527+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.entry_points")
528+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.get_aws_region")
529+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_agent_observability_enabled")
530+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_installed")
531+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches")
532+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure")
533+
def test_auto_detect_no_thirdparty_no_native_disabled(
534+
self, mock_super, mock_patches, mock_installed, mock_agent, mock_region, mock_eps
535+
):
536+
"""When no third-party instrumentors are registered, no AWS native ones should be disabled."""
537+
mock_agent.return_value = True
538+
mock_region.return_value = "us-east-1"
539+
mock_installed.return_value = False
540+
mock_eps.return_value = []
541+
542+
AwsOpenTelemetryDistro()._configure()
543+
544+
disabled = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "")
545+
self.assertNotIn("aws_crewai", disabled)
546+
self.assertNotIn("aws_langchain", disabled)
547+
self.assertNotIn("aws_llama-index", disabled)
548+
self.assertNotIn("aws_mcp", disabled)
549+
self.assertNotIn("aws_openai_agents", disabled)
550+
551+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.entry_points")
552+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.get_aws_region")
553+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_agent_observability_enabled")
554+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_installed")
555+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches")
556+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure")
557+
def test_prefer_native_disables_thirdparty(
558+
self, mock_super, mock_patches, mock_installed, mock_agent, mock_region, mock_eps
559+
):
560+
"""When AWS_AGENTIC_INSTRUMENTATION_OPT_IN=true, third-party instrumentors should be disabled."""
561+
mock_agent.return_value = True
562+
mock_region.return_value = "us-east-1"
563+
mock_installed.return_value = False
564+
os.environ["AWS_AGENTIC_INSTRUMENTATION_OPT_IN"] = "true"
565+
566+
mock_langchain_ep = MagicMock()
567+
mock_langchain_ep.name = "langchain"
568+
mock_eps.return_value = [mock_langchain_ep]
569+
570+
AwsOpenTelemetryDistro()._configure()
571+
572+
disabled = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "")
573+
self.assertIn("langchain", disabled)
574+
self.assertNotIn("aws_langchain", disabled)
575+
576+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.entry_points")
577+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.get_aws_region")
578+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_agent_observability_enabled")
579+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.is_installed")
580+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches")
581+
@patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure")
582+
def test_mutual_exclusion_appends_to_user_disabled(
583+
self, mock_super, mock_patches, mock_installed, mock_agent, mock_region, mock_eps
584+
):
585+
"""Mutual exclusion disables should append to user-provided OTEL_PYTHON_DISABLED_INSTRUMENTATIONS."""
586+
mock_agent.return_value = True
587+
mock_region.return_value = "us-east-1"
588+
mock_installed.return_value = False
589+
os.environ[OTEL_PYTHON_DISABLED_INSTRUMENTATIONS] = "my_custom_lib"
590+
591+
mock_langchain_ep = MagicMock()
592+
mock_langchain_ep.name = "langchain"
593+
mock_eps.return_value = [mock_langchain_ep]
594+
595+
AwsOpenTelemetryDistro()._configure()
596+
597+
disabled = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "")
598+
self.assertTrue(disabled.startswith("my_custom_lib,"))
599+
self.assertIn("aws_langchain", disabled)
600+
500601
def _configure_with_agent_observability(self, region="us-west-2"):
501602
with patch("amazon.opentelemetry.distro.aws_opentelemetry_distro.OpenTelemetryDistro._configure"), patch(
502603
"amazon.opentelemetry.distro.aws_opentelemetry_distro.apply_instrumentation_patches"

0 commit comments

Comments
 (0)