feat: add AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT env var to disable LLO processing (#741)

*Description of changes:*
- Add `GENAI_CONTENT_EXTRACTION_OPT_OUT` environment variable to allow
users to opt out of LLO content extraction from spans
- When set to `true` the LLO handler is bypassed and spans are exported
without content extraction
- Content extraction remains enabled by default when agent observability
is on

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: Eric Zhang <eric.zhang6811@gmail.com>
Co-authored-by: Eric Zhang <zhaez@amazon.com>

Author: liustve

CHANGELOG.md

@@ -12,6 +12,8 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t
 
 ## Unreleased
 
+- feat(agent-observability): add `AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT` env var to allow disabling LLO content extraction from spans
+  ([#741](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/741))
 - fix(mcp-instrumentation): suppress MCP `/ping` spans when agent observability is enabled
   ([#748](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/748))
 - fix(agent-observability): fall back to OTEL_EXPORTER_OTLP_ENDPOINT for unsampled spans; also export unsampled spans to non-AWS endpoints

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py

@@ -12,6 +12,7 @@
 
 AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED"
 OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS = "OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS"
+AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT = "AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT"
 
 
 def is_installed(req: str) -> bool:
@@ -39,6 +40,11 @@ def is_agent_observability_enabled() -> bool:
     return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true"
 
 
+def is_genai_content_extraction_opted_out() -> bool:
+    """Has the user opted out of GenAI content extraction from spans?"""
+    return os.environ.get(AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT, "false").lower() == "true"
+
+
 def should_add_application_signals_dimensions() -> bool:
     """Should Service and Environment Application Signals dimensions be added to EMF logs?"""
     return os.environ.get(OTEL_METRICS_ADD_APPLICATION_SIGNALS_DIMENSIONS, "true").lower() == "true"

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py

@@ -6,7 +6,7 @@
 
 from botocore.session import Session
 
-from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
+from amazon.opentelemetry.distro._utils import is_agent_observability_enabled, is_genai_content_extraction_opted_out
 from amazon.opentelemetry.distro.exporter.otlp.aws.common._aws_http_headers import _OTLP_AWS_HTTP_HEADERS
 from amazon.opentelemetry.distro.exporter.otlp.aws.common.aws_auth_session import AwsAuthSession
 from amazon.opentelemetry.distro.llo_handler import LLOHandler
@@ -77,7 +77,11 @@ def _ensure_llo_handler(self):
 
     def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
         try:
-            if is_agent_observability_enabled() and self._ensure_llo_handler():
+            if (
+                is_agent_observability_enabled()
+                and not is_genai_content_extraction_opted_out()
+                and self._ensure_llo_handler()
+            ):
                 llo_processed_spans = self._llo_handler.process_spans(spans)
                 return super().export(llo_processed_spans)
         except Exception:  # pylint: disable=broad-exception-caught

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/exporter/otlp/aws/traces/test_otlp_aws_span_exporter.py

@@ -209,3 +209,66 @@ def test_export_with_llo_processing_failure(
         result = exporter.export(spans)
 
         self.assertEqual(result, SpanExportResult.FAILURE)
+
+    @patch(
+        "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter."
+        "is_genai_content_extraction_opted_out"
+    )
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
+    def test_export_skips_llo_when_content_extraction_opted_out(
+        self, mock_llo_handler_class, mock_get_logger_provider, mock_is_enabled, mock_opted_out
+    ):
+        mock_is_enabled.return_value = True
+        mock_opted_out.return_value = True
+        mock_logger_provider = MagicMock(spec=LoggerProvider)
+        mock_get_logger_provider.return_value = mock_logger_provider
+
+        endpoint = "https://xray.us-east-1.amazonaws.com/v1/traces"
+        exporter = OTLPAwsSpanExporter(session=get_aws_session(), aws_region="us-east-1", endpoint=endpoint)
+
+        original_spans = [MagicMock(spec=ReadableSpan)]
+
+        with patch.object(OTLPSpanExporter, "export") as mock_parent_export:
+            mock_parent_export.return_value = SpanExportResult.SUCCESS
+
+            result = exporter.export(original_spans)
+
+            self.assertEqual(result, SpanExportResult.SUCCESS)
+            mock_parent_export.assert_called_once_with(original_spans)
+            mock_llo_handler_class.assert_not_called()
+
+    @patch(
+        "amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter."
+        "is_genai_content_extraction_opted_out"
+    )
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.is_agent_observability_enabled")
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.get_logger_provider")
+    @patch("amazon.opentelemetry.distro.exporter.otlp.aws.traces.otlp_aws_span_exporter.LLOHandler")
+    def test_export_processes_llo_when_content_extraction_not_opted_out(
+        self, mock_llo_handler_class, mock_get_logger_provider, mock_is_enabled, mock_opted_out
+    ):
+        mock_is_enabled.return_value = True
+        mock_opted_out.return_value = False
+        mock_logger_provider = MagicMock(spec=LoggerProvider)
+        mock_get_logger_provider.return_value = mock_logger_provider
+
+        mock_llo_handler = MagicMock()
+        mock_llo_handler_class.return_value = mock_llo_handler
+
+        endpoint = "https://xray.us-east-1.amazonaws.com/v1/traces"
+        exporter = OTLPAwsSpanExporter(session=get_aws_session(), aws_region="us-east-1", endpoint=endpoint)
+
+        original_spans = [MagicMock(spec=ReadableSpan)]
+        processed_spans = [MagicMock(spec=ReadableSpan)]
+        mock_llo_handler.process_spans.return_value = processed_spans
+
+        with patch.object(OTLPSpanExporter, "export") as mock_parent_export:
+            mock_parent_export.return_value = SpanExportResult.SUCCESS
+
+            result = exporter.export(original_spans)
+
+            self.assertEqual(result, SpanExportResult.SUCCESS)
+            mock_llo_handler.process_spans.assert_called_once_with(original_spans)
+            mock_parent_export.assert_called_once_with(processed_spans)

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_utils.py

@@ -8,28 +8,33 @@
 
 from amazon.opentelemetry.distro._utils import (
     AGENT_OBSERVABILITY_ENABLED,
+    AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT,
     get_aws_region,
     get_aws_session,
     is_agent_observability_enabled,
+    is_genai_content_extraction_opted_out,
     is_installed,
 )
 
 
 class TestUtils(TestCase):
     def setUp(self):
-        # Store original env var if it exists
         self.original_env = os.environ.get(AGENT_OBSERVABILITY_ENABLED)
-        # Clear it to ensure clean state
+        self.original_opt_out_env = os.environ.get(AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT)
         if AGENT_OBSERVABILITY_ENABLED in os.environ:
             del os.environ[AGENT_OBSERVABILITY_ENABLED]
+        if AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT in os.environ:
+            del os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT]
 
     def tearDown(self):
-        # First clear the env var
         if AGENT_OBSERVABILITY_ENABLED in os.environ:
             del os.environ[AGENT_OBSERVABILITY_ENABLED]
-        # Then restore original if it existed
+        if AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT in os.environ:
+            del os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT]
         if self.original_env is not None:
             os.environ[AGENT_OBSERVABILITY_ENABLED] = self.original_env
+        if self.original_opt_out_env is not None:
+            os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = self.original_opt_out_env
 
     def test_is_installed_package_not_found(self):
         """Test is_installed returns False when package is not found"""
@@ -173,3 +178,26 @@ def test_get_aws_region_with_aws_default_region_env(self):
         self.assertEqual(region, "eu-west-1")
 
         os.environ.pop("AWS_DEFAULT_REGION", None)
+
+    def test_is_genai_content_extraction_opted_out_various_values(self):
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "true"
+        self.assertTrue(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "True"
+        self.assertTrue(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "TRUE"
+        self.assertTrue(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "false"
+        self.assertFalse(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = "False"
+        self.assertFalse(is_genai_content_extraction_opted_out())
+
+        os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT] = ""
+        self.assertFalse(is_genai_content_extraction_opted_out())
+
+        if AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT in os.environ:
+            del os.environ[AWS_GENAI_CONTENT_EXTRACTION_OPT_OUT]
+        self.assertFalse(is_genai_content_extraction_opted_out())