From 8d94cb59adbfa0591dcc9ff2f62432ab2eddc395 Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Fri, 24 Apr 2026 19:04:20 -0700 Subject: [PATCH 1/4] fix unsampled span export to fall back to OTEL_EXPORTER_OTLP_ENDPOINT --- .../distro/aws_opentelemetry_configurator.py | 13 ++++++++++--- .../distro/test_aws_opentelementry_configurator.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py index 81f87ba3b..a0e45eae6 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py @@ -330,12 +330,19 @@ def _export_unsampled_span_for_agent_observability(trace_provider: TracerProvide if not is_agent_observability_enabled(): return - traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) - if traces_endpoint and _is_aws_otlp_endpoint(traces_endpoint, XRAY_SERVICE): + traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) or os.environ.get( + "OTEL_EXPORTER_OTLP_ENDPOINT" + ) + if not traces_endpoint: + return + + if _is_aws_otlp_endpoint(traces_endpoint, XRAY_SERVICE): endpoint, region = _extract_endpoint_and_region_from_otlp_endpoint(traces_endpoint) span_exporter = _create_aws_otlp_exporter(endpoint=endpoint, service=XRAY_SERVICE, region=region) + else: + span_exporter = OTLPSpanExporter(endpoint=traces_endpoint) - trace_provider.add_span_processor(BatchUnsampledSpanProcessor(span_exporter=span_exporter)) + trace_provider.add_span_processor(BatchUnsampledSpanProcessor(span_exporter=span_exporter)) def _is_defer_to_workers_enabled(): diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py index 265a341a2..25fccb0b2 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py @@ -1102,6 +1102,18 @@ def test_export_unsampled_span_for_agent_observability(self): os.environ.pop("AGENT_OBSERVABILITY_ENABLED", None) os.environ.pop("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", None) + mock_tracer_provider.reset_mock() + + os.environ["AGENT_OBSERVABILITY_ENABLED"] = "true" + os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "http://localhost:4318" + _export_unsampled_span_for_agent_observability(mock_tracer_provider, Resource.get_empty()) + self.assertEqual(mock_tracer_provider.add_span_processor.call_count, 1) + processor = mock_tracer_provider.add_span_processor.call_args_list[0].args[0] + self.assertIsInstance(processor, BatchUnsampledSpanProcessor) + + os.environ.pop("AGENT_OBSERVABILITY_ENABLED", None) + os.environ.pop("OTEL_EXPORTER_OTLP_ENDPOINT", None) + # pylint: disable=no-self-use def test_export_unsampled_span_for_agent_observability_uses_aws_exporter(self): """Test that OTLPAwsSpanExporter is used for AWS endpoints""" From fe38f99141c52acb0b27a52c1e146f701247a17f Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Fri, 24 Apr 2026 19:04:54 -0700 Subject: [PATCH 2/4] add changelog entry for PR #738 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ac28f986..373e5c18d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t ## Unreleased +- fix(agent-observability): fall back to OTEL_EXPORTER_OTLP_ENDPOINT for unsampled spans + ([#738](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/738)) - feat: auto-detect and mutually exclude AWS native vs third-party agentic instrumentors; add `AWS_AGENTIC_INSTRUMENTATION_OPT_IN` env var to override auto-detection ([#729](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/729)) - fix(lambda-layer): align context propagation with JS — delegate to global propagator so W3C traceparent is no longer ignored when X-Ray active tracing is enabled From 4bc55cbb9eec41f71c34db6755331e3bb980c025 Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Fri, 24 Apr 2026 19:09:33 -0700 Subject: [PATCH 3/4] append /v1/traces to base OTEL_EXPORTER_OTLP_ENDPOINT fallback --- .../distro/aws_opentelemetry_configurator.py | 8 +++++--- .../distro/test_aws_opentelementry_configurator.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py index a0e45eae6..bd19c8f23 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py @@ -330,9 +330,11 @@ def _export_unsampled_span_for_agent_observability(trace_provider: TracerProvide if not is_agent_observability_enabled(): return - traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) or os.environ.get( - "OTEL_EXPORTER_OTLP_ENDPOINT" - ) + traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) + if not traces_endpoint: + base_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") + if base_endpoint: + traces_endpoint = base_endpoint.rstrip("/") + "/v1/traces" if not traces_endpoint: return diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py index 25fccb0b2..a12e355b3 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py @@ -1110,6 +1110,7 @@ def test_export_unsampled_span_for_agent_observability(self): self.assertEqual(mock_tracer_provider.add_span_processor.call_count, 1) processor = mock_tracer_provider.add_span_processor.call_args_list[0].args[0] self.assertIsInstance(processor, BatchUnsampledSpanProcessor) + self.assertEqual(processor.span_exporter._endpoint, "http://localhost:4318/v1/traces") os.environ.pop("AGENT_OBSERVABILITY_ENABLED", None) os.environ.pop("OTEL_EXPORTER_OTLP_ENDPOINT", None) From 2f15fb0774b1b03557d96bac40613015020195b1 Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Mon, 27 Apr 2026 10:59:17 -0700 Subject: [PATCH 4/4] address PR review comments: use SDK constant, add test coverage --- CHANGELOG.md | 2 +- .../distro/aws_opentelemetry_configurator.py | 3 ++- .../test_aws_opentelementry_configurator.py | 24 +++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 373e5c18d..a9781a80a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t ## Unreleased -- fix(agent-observability): fall back to OTEL_EXPORTER_OTLP_ENDPOINT for unsampled spans +- fix(agent-observability): fall back to OTEL_EXPORTER_OTLP_ENDPOINT for unsampled spans; also export unsampled spans to non-AWS endpoints ([#738](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/738)) - feat: auto-detect and mutually exclude AWS native vs third-party agentic instrumentors; add `AWS_AGENTIC_INSTRUMENTATION_OPT_IN` env var to override auto-detection ([#729](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/729)) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py index bd19c8f23..f3906c36f 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py @@ -64,6 +64,7 @@ from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, ConsoleLogRecordExporter, LogRecordExporter from opentelemetry.sdk.environment_variables import ( _OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, + OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_METRICS_PROTOCOL, OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_TRACES_SAMPLER_ARG, @@ -332,7 +333,7 @@ def _export_unsampled_span_for_agent_observability(trace_provider: TracerProvide traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) if not traces_endpoint: - base_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") + base_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_ENDPOINT) if base_endpoint: traces_endpoint = base_endpoint.rstrip("/") + "/v1/traces" if not traces_endpoint: diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py index a12e355b3..b1c218adb 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py @@ -1115,6 +1115,30 @@ def test_export_unsampled_span_for_agent_observability(self): os.environ.pop("AGENT_OBSERVABILITY_ENABLED", None) os.environ.pop("OTEL_EXPORTER_OTLP_ENDPOINT", None) + mock_tracer_provider.reset_mock() + + os.environ["AGENT_OBSERVABILITY_ENABLED"] = "true" + os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "https://xray.us-west-2.amazonaws.com" + _export_unsampled_span_for_agent_observability(mock_tracer_provider, Resource.get_empty()) + self.assertEqual(mock_tracer_provider.add_span_processor.call_count, 1) + + os.environ.pop("AGENT_OBSERVABILITY_ENABLED", None) + os.environ.pop("OTEL_EXPORTER_OTLP_ENDPOINT", None) + + mock_tracer_provider.reset_mock() + + os.environ["AGENT_OBSERVABILITY_ENABLED"] = "true" + os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = "https://xray.us-east-1.amazonaws.com/v1/traces" + os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "http://localhost:4318" + _export_unsampled_span_for_agent_observability(mock_tracer_provider, Resource.get_empty()) + self.assertEqual(mock_tracer_provider.add_span_processor.call_count, 1) + processor = mock_tracer_provider.add_span_processor.call_args_list[0].args[0] + self.assertIsInstance(processor, BatchUnsampledSpanProcessor) + + os.environ.pop("AGENT_OBSERVABILITY_ENABLED", None) + os.environ.pop("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", None) + os.environ.pop("OTEL_EXPORTER_OTLP_ENDPOINT", None) + # pylint: disable=no-self-use def test_export_unsampled_span_for_agent_observability_uses_aws_exporter(self): """Test that OTLPAwsSpanExporter is used for AWS endpoints"""