Skip to content

Commit 00ecc61

Browse files
wangzleiwangzlei
committed
feat: auto-detect mutual exclusion between AWS native and third-party agentic instrumentors
When AGENT_OBSERVABILITY_ENABLED=true, auto-detect registered third-party instrumentors (e.g. OpenInference) and disable conflicting AWS native ones to prevent double instrumentation. Add AWS_AGENTIC_INSTRUMENTATION_OPT_IN env var to let users override auto-detection and force AWS native instrumentors over third-party ones.
1 parent 537ac16 commit 00ecc61

6 files changed

Lines changed: 210 additions & 29 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ pip-log.txt
3030
# Unit test / coverage reports
3131
coverage.xml
3232
.coverage
33+
.coverage.*
3334
.nox
3435
.tox
3536
.cache

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t
1212

1313
## Unreleased
1414

15+
- feat: auto-detect and mutually exclude AWS native vs third-party agentic instrumentors; add `AWS_AGENTIC_INSTRUMENTATION_OPT_IN` env var to override auto-detection
16+
([#729](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/729))
1517
- feat: support environment-configured endpoint visibility for HTTP operation names
1618
([#718](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/718))
1719
- fix(lambda-layer): Disable all agentic instrumentation in Lambda by default

‎aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED"
1414
OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS = "OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS"
1515

16+
1617
def is_installed(req: str) -> bool:
1718
"""Is the given required package installed?"""
1819
req = Requirement(req)

‎aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_distro.py‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,20 @@ def _check_otel_version_compatibility():
9898
OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION,
9999
OTEL_EXPORTER_OTLP_PROTOCOL,
100100
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
101-
OTEL_TRACES_SAMPLER,
102101
)
102+
from opentelemetry.util._importlib_metadata import entry_points
103103

104104
_logger: Logger = getLogger(__name__)
105105
# Suppress configurator warnings from auto-instrumentation
106106
_load._logger.setLevel(LEVELS.get(os.environ.get(OTEL_PYTHON_LOG_LEVEL, "error").lower(), ERROR))
107107

108108

109+
# When set to "true", opt in to AWS agentic instrumentors over third-party ones (e.g. OpenInference),
110+
# even if both are installed. By default, auto-detection prefers third-party when present.
111+
AWS_AGENTIC_INSTRUMENTATION_OPT_IN = "AWS_AGENTIC_INSTRUMENTATION_OPT_IN"
112+
113+
# Maps third-party instrumentor entry point names to their AWS native equivalents.
114+
# Used for mutual exclusion: only one side instruments each library at a time.
109115
_THIRDPARTY_TO_AWS_NATIVE = {
110116
"crewai": "aws_crewai",
111117
"langchain": "aws_langchain",
@@ -115,6 +121,10 @@ def _check_otel_version_compatibility():
115121
"openai_agents": "aws_openai_agents",
116122
}
117123

124+
# Dist names owned by ADOT that register entry points with the same names as third-party ones.
125+
# These are excluded from third-party detection to avoid false positives.
126+
_ADOT_OWNED_DISTS = {"opentelemetry-instrumentation-openai-agents-v2"}
127+
118128

119129
class AwsOpenTelemetryDistro(OpenTelemetryDistro):
120130
def _configure(self, **kwargs):
@@ -207,21 +217,50 @@ def _configure(self, **kwargs):
207217
"Please set AWS_REGION environment variable or configure OTLP endpoints manually."
208218
)
209219

210-
os.environ.setdefault(OTEL_TRACES_SAMPLER, "parentbased_always_on")
211220
os.environ.setdefault(
212221
OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
213222
"http,sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector,"
214223
"urllib3,requests,system_metrics,google-genai,jinja2",
215224
)
225+
self._apply_agentic_instrumentation_mutual_exclusion()
216226
os.environ.setdefault(OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "true")
217227
os.environ.setdefault(OTEL_PYTHON_LOG_CORRELATION, "true")
218228
os.environ.setdefault(APPLICATION_SIGNALS_ENABLED_CONFIG, "false")
219229
os.environ.setdefault(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "false")
220230
os.environ.setdefault("CREWAI_DISABLE_TELEMETRY", "true")
221231

222-
223232
super(AwsOpenTelemetryDistro, self)._configure()
224233

225234
if kwargs.get("apply_patches", True):
226235
apply_instrumentation_patches()
227236

237+
@staticmethod
238+
def _apply_agentic_instrumentation_mutual_exclusion():
239+
"""Mutual exclusion between AWS native and third-party agentic instrumentors.
240+
241+
AWS_AGENTIC_INSTRUMENTATION_OPT_IN=true overrides auto-detection
242+
and forces AWS native instrumentors, disabling third-party ones.
243+
Default (auto-detect): if a third-party instrumentor is registered, disable the AWS native one.
244+
"""
245+
prefer_native = os.environ.get(AWS_AGENTIC_INSTRUMENTATION_OPT_IN, "false").lower() == "true"
246+
registered = {
247+
ep.name
248+
for ep in entry_points(group="opentelemetry_instrumentor")
249+
if not (ep.dist and ep.dist.name in _ADOT_OWNED_DISTS)
250+
}
251+
disable_native = []
252+
disable_third_party = []
253+
for third_party, aws_native in _THIRDPARTY_TO_AWS_NATIVE.items():
254+
if third_party in registered:
255+
if prefer_native:
256+
disable_third_party.append(third_party)
257+
else:
258+
disable_native.append(aws_native)
259+
else:
260+
disable_third_party.append(third_party)
261+
262+
# Force-disable conflicting instrumentors to prevent double instrumentation.
263+
to_disable = disable_native if not prefer_native else disable_third_party
264+
if to_disable:
265+
existing = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "")
266+
os.environ[OTEL_PYTHON_DISABLED_INSTRUMENTATIONS] = existing + "," + ",".join(dict.fromkeys(to_disable))

‎aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,7 @@ def test_aws_headers_applied(self):
5050
self.assertEqual(exporter._session.headers["X-Custom-Header"], "custom-value")
5151
self.assertIn("User-Agent", exporter._session.headers)
5252

53-
@patch(
54-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
55-
)
53+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
5654
def test_ensure_llo_handler_when_disabled(self, mock_is_enabled):
5755
# Test _ensure_llo_handler when agent observability is disabled
5856
mock_is_enabled.return_value = False
@@ -66,9 +64,7 @@ def test_ensure_llo_handler_when_disabled(self, mock_is_enabled):
6664
mock_is_enabled.assert_called_once()
6765

6866
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
69-
@patch(
70-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
71-
)
67+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
7268
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
7369
def test_ensure_llo_handler_lazy_initialization(
7470
self, mock_llo_handler_class, mock_is_enabled, mock_get_logger_provider
@@ -102,9 +98,7 @@ def test_ensure_llo_handler_lazy_initialization(
10298
mock_get_logger_provider.assert_not_called()
10399

104100
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
105-
@patch(
106-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
107-
)
101+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
108102
def test_ensure_llo_handler_with_existing_logger_provider(self, mock_is_enabled, mock_get_logger_provider):
109103
# Test when logger_provider is already provided
110104
mock_is_enabled.return_value = True
@@ -129,9 +123,7 @@ def test_ensure_llo_handler_with_existing_logger_provider(self, mock_is_enabled,
129123
mock_get_logger_provider.assert_not_called()
130124

131125
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
132-
@patch(
133-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
134-
)
126+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
135127
def test_ensure_llo_handler_get_logger_provider_fails(self, mock_is_enabled, mock_get_logger_provider):
136128
# Test when get_logger_provider raises exception
137129
mock_is_enabled.return_value = True
@@ -145,9 +137,7 @@ def test_ensure_llo_handler_get_logger_provider_fails(self, mock_is_enabled, moc
145137
self.assertFalse(result)
146138
self.assertIsNone(exporter._llo_handler)
147139

148-
@patch(
149-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
150-
)
140+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
151141
def test_export_with_llo_disabled(self, mock_is_enabled):
152142
# Test export when LLO is disabled
153143
mock_is_enabled.return_value = False
@@ -166,9 +156,7 @@ def test_export_with_llo_disabled(self, mock_is_enabled):
166156
mock_parent_export.assert_called_once_with(spans)
167157
self.assertIsNone(exporter._llo_handler)
168158

169-
@patch(
170-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
171-
)
159+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
172160
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
173161
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
174162
def test_export_with_llo_enabled(self, mock_llo_handler_class, mock_get_logger_provider, mock_is_enabled):
@@ -198,9 +186,7 @@ def test_export_with_llo_enabled(self, mock_llo_handler_class, mock_get_logger_p
198186
mock_llo_handler.process_spans.assert_called_once_with(original_spans)
199187
mock_parent_export.assert_called_once_with(processed_spans)
200188

201-
@patch(
202-
"amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled"
203-
)
189+
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
204190
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
205191
@patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
206192
def test_export_with_llo_processing_failure(

0 commit comments

Comments
 (0)