Merge branch 'main' into converse-contract-tests

Author: liustve

CHANGELOG.md

@@ -12,6 +12,10 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t
 
 ## Unreleased
 
+- feat(agent-observability): add `AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT` env var to allow disabling LLO content extraction from spans
+  ([#741](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/741))
+- fix(mcp-instrumentation): suppress MCP `/ping` spans when agent observability is enabled
+  ([#748](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/748))
 - fix(agent-observability): fall back to OTEL_EXPORTER_OTLP_ENDPOINT for unsampled spans; also export unsampled spans to non-AWS endpoints
   ([#738](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/738))
 - feat: auto-detect and mutually exclude AWS native vs third-party agentic instrumentors; add `AWS_AGENTIC_INSTRUMENTATION_OPT_IN` env var to override auto-detection

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py

@@ -12,6 +12,7 @@
 
 AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED"
 OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS = "OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS"
+AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT = "AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT"
 
 
 def is_installed(req: str) -> bool:
@@ -39,6 +40,11 @@ def is_agent_observability_enabled() -> bool:
     return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true"
 
 
+def is_genai_content_extraction_opted_out() -> bool:
+    """Has the user opted out of GenAI content extraction from spans?"""
+    return os.environ.get(AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT, "false").lower() == "true"
+
+
 def should_add_application_signals_dimensions() -> bool:
     """Should Service and Environment Application Signals dimensions be added to EMF logs?"""
     return os.environ.get(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "true").lower() == "true"

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py

@@ -6,7 +6,7 @@
 
 from botocore.session import Session
 
-from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
+from amazon.opentelemetry.distro._utils import is_agent_observability_enabled, is_genai_content_extraction_opted_out
 from amazon.opentelemetry.distro.exporter.otlp.aws.common._aws_http_headers import _OTLP_AWS_HTTP_HEADERS
 from amazon.opentelemetry.distro.exporter.otlp.aws.common.aws_auth_session import AwsAuthSession
 from amazon.opentelemetry.distro.llo_handler import LLOHandler
@@ -77,7 +77,11 @@ def _ensure_llo_handler(self):
 
     def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
         try:
-            if is_agent_observability_enabled() and self._ensure_llo_handler():
+            if (
+                is_agent_observability_enabled()
+                and not is_genai_content_extraction_opted_out()
+                and self._ensure_llo_handler()
+            ):
                 llo_processed_spans = self._llo_handler.process_spans(spans)
                 return super().export(llo_processed_spans)
         except Exception:  # pylint: disable=broad-exception-caught

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/instrumentation/mcp/_wrappers.py

@@ -8,6 +8,7 @@
 from typing import Any, Callable, Coroutine, Dict, Optional, Tuple
 from urllib.parse import urlparse
 
+from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
 from amazon.opentelemetry.distro.instrumentation.common.instrumentation_utils import serialize_to_json_string
 from opentelemetry import context, trace
 from opentelemetry.instrumentation.utils import suppress_http_instrumentation
@@ -60,17 +61,22 @@ def __init__(self, tracer: trace.Tracer, **kwargs: Any) -> None:
         self._should_suppress_http_spans = (
             os.environ.get(OTEL_MCP_SUPPRESS_HTTP_INSTRUMENTATION, "true").lower() == "true"
         )
+        self._agent_observability_enabled = is_agent_observability_enabled()
 
-    @staticmethod
-    def _should_suppress_mcp_span(message: Any) -> bool:
+    def _should_suppress_mcp_span(self, message: Any) -> bool:
         from mcp import types  # pylint: disable=import-outside-toplevel
 
         if isinstance(
             message, (types.ClientRequest, types.ClientNotification, types.ServerRequest, types.ServerNotification)
         ):
             message = message.root
         # noisy spans most of the time
-        return isinstance(message, (types.InitializeRequest, types.InitializedNotification))
+        if isinstance(message, (types.InitializeRequest, types.InitializedNotification)):
+            return True
+        # MCP servers hosted on AgentCore get frequent /ping health checks
+        if self._agent_observability_enabled and isinstance(message, types.PingRequest):
+            return True
+        return False
 
     @staticmethod
     def _set_mcp_attributes(span: trace.Span, message: Any, request_id: Optional[int]) -> None:

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py

@@ -209,3 +209,66 @@ def test_export_with_llo_processing_failure(
         result = exporter.export(spans)
 
         self.assertEqual(result, SpanExportResult.FAILURE)
+
+    @patch(
+        "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter."
+        "is_genai_content_extraction_opted_out"
+    )
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
+    def test_export_skips_llo_when_content_extraction_opted_out(
+        self, mock_llo_handler_class, mock_get_logger_provider, mock_is_enabled, mock_opted_out
+    ):
+        mock_is_enabled.return_value = True
+        mock_opted_out.return_value = True
+        mock_logger_provider = MagicMock(spec=LoggerProvider)
+        mock_get_logger_provider.return_value = mock_logger_provider
+
+        endpoint = "https://xray.us-east-1.amazonaws.com/v1/traces"
+        exporter = OTLPAwsSpanExporter(session=get_aws_session(), aws_region="us-east-1", endpoint=endpoint)
+
+        original_spans = [MagicMock(spec=ReadableSpan)]
+
+        with patch.object(OTLPSpanExporter, "export") as mock_parent_export:
+            mock_parent_export.return_value = SpanExportResult.SUCCESS
+
+            result = exporter.export(original_spans)
+
+            self.assertEqual(result, SpanExportResult.SUCCESS)
+            mock_parent_export.assert_called_once_with(original_spans)
+            mock_llo_handler_class.assert_not_called()
+
+    @patch(
+        "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter."
+        "is_genai_content_extraction_opted_out"
+    )
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
+    def test_export_processes_llo_when_content_extraction_not_opted_out(
+        self, mock_llo_handler_class, mock_get_logger_provider, mock_is_enabled, mock_opted_out
+    ):
+        mock_is_enabled.return_value = True
+        mock_opted_out.return_value = False
+        mock_logger_provider = MagicMock(spec=LoggerProvider)
+        mock_get_logger_provider.return_value = mock_logger_provider
+
+        mock_llo_handler = MagicMock()
+        mock_llo_handler_class.return_value = mock_llo_handler
+
+        endpoint = "https://xray.us-east-1.amazonaws.com/v1/traces"
+        exporter = OTLPAwsSpanExporter(session=get_aws_session(), aws_region="us-east-1", endpoint=endpoint)
+
+        original_spans = [MagicMock(spec=ReadableSpan)]
+        processed_spans = [MagicMock(spec=ReadableSpan)]
+        mock_llo_handler.process_spans.return_value = processed_spans
+
+        with patch.object(OTLPSpanExporter, "export") as mock_parent_export:
+            mock_parent_export.return_value = SpanExportResult.SUCCESS
+
+            result = exporter.export(original_spans)
+
+            self.assertEqual(result, SpanExportResult.SUCCESS)
+            mock_llo_handler.process_spans.assert_called_once_with(original_spans)
+            mock_parent_export.assert_called_once_with(processed_spans)

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/instrumentation/mcp/test_mcp_instrumentor.py

@@ -473,6 +473,27 @@ async def run(session):
                     finally:
                         HTTPXClientInstrumentor().uninstrument()
 
+    def test_ping_span_suppression(self):
+        for agent_obs_enabled, expect_suppressed in [("true", True), ("false", False)]:
+            with self.subTest(agent_observability=agent_obs_enabled):
+                self.instrumentor.uninstrument()
+                self.span_exporter.clear()
+                with unittest.mock.patch.dict(os.environ, {"AGENT_OBSERVABILITY_ENABLED": agent_obs_enabled}):
+                    self.instrumentor.instrument(tracer_provider=self.tracer_provider, propagators=self.propagator)
+                    self.server = self._create_server()
+
+                    async def run(session):
+                        await session.send_ping()
+
+                    asyncio.run(self._run_inprocess(run))
+                    spans = self.span_exporter.get_finished_spans()
+
+                    ping_spans = [s for s in spans if "ping" in s.name.lower()]
+                    if expect_suppressed:
+                        self.assertEqual(len(ping_spans), 0, "Ping spans should be suppressed")
+                    else:
+                        self.assertGreater(len(ping_spans), 0, "Ping spans should not be suppressed")
+
     def test_mcp_respects_active_parent_span(self):
         tracer = get_tracer("test", tracer_provider=self.tracer_provider)
 

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_utils.py

@@ -8,28 +8,33 @@
 
 from amazon.opentelemetry.distro._utils import (
     AGENT_OBSERVABILITY_ENABLED,
+    AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT,
     get_aws_region,
     get_aws_session,
     is_agent_observability_enabled,
+    is_genai_content_extraction_opted_out,
     is_installed,
 )
 
 
 class TestUtils(TestCase):
     def setUp(self):
-        # Store original env var if it exists
         self.original_env = os.environ.get(AGENT_OBSERVABILITY_ENABLED)
-        # Clear it to ensure clean state
+        self.original_opt_out_env = os.environ.get(AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT)
         if AGENT_OBSERVABILITY_ENABLED in os.environ:
             del os.environ[AGENT_OBSERVABILITY_ENABLED]
+        if AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT in os.environ:
+            del os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT]
 
     def tearDown(self):
-        # First clear the env var
         if AGENT_OBSERVABILITY_ENABLED in os.environ:
             del os.environ[AGENT_OBSERVABILITY_ENABLED]
-        # Then restore original if it existed
+        if AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT in os.environ:
+            del os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT]
         if self.original_env is not None:
             os.environ[AGENT_OBSERVABILITY_ENABLED] = self.original_env
+        if self.original_opt_out_env is not None:
+            os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = self.original_opt_out_env
 
     def test_is_installed_package_not_found(self):
         """Test is_installed returns False when package is not found"""
@@ -173,3 +178,26 @@ def test_get_aws_region_with_aws_default_region_env(self):
         self.assertEqual(region, "eu-west-1")
 
         os.environ.pop("AWS_DEFAULT_REGION", None)
+
+    def test_is_genai_content_extraction_opted_out_various_values(self):
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "true"
+        self.assertTrue(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "True"
+        self.assertTrue(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "TRUE"
+        self.assertTrue(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "false"
+        self.assertFalse(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "False"
+        self.assertFalse(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = ""
+        self.assertFalse(is_genai_content_extraction_opted_out())
+
+        if AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT in os.environ:
+            del os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT]
+        self.assertFalse(is_genai_content_extraction_opted_out())