[monitor opentelemetry exporter] Handle ServiceResponseError as retryable to prevent telemetry loss (#47870)

* Handle ServiceResponseError as retryable to prevent telemetry loss

ServiceResponseError (e.g. read timeout) was not caught by the exporter's
_transmit method. Since ServiceResponseError and ServiceRequestError are
sibling classes (both extend AzureError directly), a read timeout fell
through to the generic 'except Exception' handler, which marked the
batch as FAILED_NOT_RETRYABLE — permanently dropping the telemetry
instead of persisting it to local storage for retry.

This change adds ServiceResponseError to the import and catch chain,
treating it as FAILED_RETRYABLE so that envelopes are written to disk
and retried on the next successful export cycle, consistent with the
existing ServiceRequestError handling and the azure-core docstring
which states these errors 'can be retried for idempotent or safe
operations'.

* Cleanup verbose comment

* Retrigger CI/CD pipeline

* Add CHANGELOG

* Add ServiceResponseTimeoutError to the classification categories

* Add tests and expand network exceptions category

* Fix format

* Retrigger CI/CD pipeline

---------

Co-authored-by: Radhika Gupta <guptaradhika@microsoft.com>

Author: hectorhdzg

sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md

@@ -3,6 +3,8 @@
 ## 1.0.0b56 (Unreleased)
 
 ### Features Added
+- Retry payloads for which request is sent successfully but no response is received, per [SPEC.](https://github.com/aep-health-and-standards/Telemetry-Collection-Spec/pull/1018)
+  ([#47870](https://github.com/Azure/azure-sdk-for-python/pull/47870))
 
 ### Breaking Changes
 

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py

@@ -12,7 +12,7 @@
 from urllib.parse import urlparse
 import psutil
 
-from azure.core.exceptions import HttpResponseError, ServiceRequestError
+from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError
 from azure.core.pipeline.policies import (
     ContentDecodePolicy,
     HttpLoggingPolicy,
@@ -598,6 +598,21 @@ def _transmit(self, envelopes: List[TelemetryItem], _skip_rate_limit: bool = Fal
                         exc_type = request_error.__class__.__name__  # type: ignore
                     _update_requests_map(_REQ_EXCEPTION_NAME[1], value=exc_type)
                 result = ExportResult.FAILED_RETRYABLE
+            except ServiceResponseError as response_error:
+                # The request was sent but the client failed to receive a response
+                # (e.g. read timeout).
+                logger.warning("Retrying due to server response error: %s.", response_error.message)
+
+                # Track retry items in customer sdkstats for client-side exceptions
+                if self._should_collect_customer_sdkstats():
+                    track_retry_items(envelopes, response_error)
+
+                if self._should_collect_stats():
+                    exc_type = response_error.exc_type
+                    if exc_type is None or exc_type is type(None):  # pylint: disable=unidiomatic-typecheck
+                        exc_type = response_error.__class__.__name__  # type: ignore
+                    _update_requests_map(_REQ_EXCEPTION_NAME[1], value=exc_type)
+                result = ExportResult.FAILED_RETRYABLE
             except Exception as ex:
                 logger.exception(  # pylint: disable=do-not-log-exceptions-if-not-debug, do-not-use-logging-exception
                     "Envelopes could not be exported and are not retryable: %s.", ex

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/customer/_utils.py

@@ -5,7 +5,12 @@
 
 # mypy: disable-error-code="import-untyped"
 from requests import ReadTimeout, Timeout  # pylint: disable=networking-import-outside-azure-core-transport
-from azure.core.exceptions import ServiceRequestTimeoutError
+from azure.core.exceptions import (
+    ServiceRequestTimeoutError,
+    ServiceResponseTimeoutError,
+    ServiceResponseError,
+    ServiceRequestError,
+)
 from azure.monitor.opentelemetry.exporter._constants import (
     _REQUEST,
     RetryCode,
@@ -95,11 +100,14 @@ def _determine_client_retry_code(
     """
     timeout_exception_types = (
         ServiceRequestTimeoutError,
+        ServiceResponseTimeoutError,
         ReadTimeout,
         TimeoutError,
         Timeout,
     )
     network_exception_types = (
+        ServiceRequestError,
+        ServiceResponseError,
         ConnectionError,
         OSError,
     )

sdk/monitor/azure-monitor-opentelemetry-exporter/tests/customer_sdk_stats/test_utlities.py

@@ -7,7 +7,13 @@
 from datetime import datetime
 
 from requests.exceptions import ConnectionError, ReadTimeout, Timeout
-from azure.core.exceptions import ServiceRequestTimeoutError, HttpResponseError
+from azure.core.exceptions import (
+    ServiceRequestTimeoutError,
+    HttpResponseError,
+    ServiceResponseTimeoutError,
+    ServiceResponseError,
+    ServiceRequestError,
+)
 
 from azure.monitor.opentelemetry.exporter._generated.exporter.models import (
     TelemetryItem,
@@ -310,6 +316,7 @@ def test_determine_client_retry_code_timeout_errors(self):
             ReadTimeout("Read timeout"),
             TimeoutError("Generic timeout"),
             Timeout("Requests timeout"),
+            ServiceResponseTimeoutError("Service response timeout"),
         ]
 
         for error in timeout_errors:
@@ -343,6 +350,8 @@ def test_determine_client_retry_code_network_errors(self):
         network_errors = [
             ConnectionError("Connection failed"),
             OSError("OS error occurred"),
+            ServiceResponseError("Service response error"),
+            ServiceRequestError("Service request error"),
         ]
 
         for error in network_errors:

sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py

@@ -6,7 +6,7 @@
 from unittest import mock
 from datetime import datetime
 
-from azure.core.exceptions import HttpResponseError, ServiceRequestError
+from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError
 from azure.monitor.opentelemetry.exporter.export._base import (
     BaseExporter,
     ExportResult,
@@ -215,6 +215,16 @@ def test_transmit_service_request_error_customer_sdkstats_track_retry_items(self
         track_retry_mock.assert_called_once()
         self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
 
+    @mock.patch("azure.monitor.opentelemetry.exporter.export._base.track_retry_items")
+    def test_transmit_service_response_error_customer_sdkstats_track_retry_items(self, track_retry_mock):
+        """Test that _track_retry_items is called on ServiceResponseError (e.g. read timeout)"""
+        exporter = self._create_exporter_with_customer_sdkstats_enabled()
+        with mock.patch.object(AzureMonitorClient, "track", side_effect=ServiceResponseError("Read timed out")):
+            result = exporter._transmit(self._envelopes_to_export)
+
+        track_retry_mock.assert_called_once()
+        self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
+
     @mock.patch("azure.monitor.opentelemetry.exporter.export._base.track_dropped_items")
     def test_transmit_general_exception_customer_sdkstats_track_dropped_items(self, track_dropped_mock):
         """Test that _track_dropped_items is called on general exceptions"""

sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py

@@ -7,7 +7,7 @@
 from unittest import mock
 from datetime import datetime, timedelta, timezone
 
-from azure.core.exceptions import HttpResponseError, ServiceRequestError
+from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError
 from azure.core.pipeline.transport import HttpResponse
 from azure.monitor.opentelemetry.exporter.export._base import (
     _get_auth_policy,
@@ -861,6 +861,11 @@ def test_transmit_request_error(self):
             result = self._base._transmit(self._envelopes_to_export)
         self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
 
+    def test_transmit_response_error(self):
+        with mock.patch.object(AzureMonitorClient, "track", throw(ServiceResponseError, message="Read timed out")):
+            result = self._base._transmit(self._envelopes_to_export)
+        self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
+
     def test_transmission_200(self):
         with mock.patch.object(AzureMonitorClient, "track") as post:
             post.return_value = TrackResponse(
@@ -1180,6 +1185,25 @@ def test_transmit_request_error_statsbeat(self, stats_mock):
         self.assertEqual(_REQUESTS_MAP["count"], 1)
         self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
 
+    @mock.patch.dict(
+        os.environ,
+        {
+            "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "false",
+            "APPLICATIONINSIGHTS_SDKSTATS_DISABLED": "false",
+        },
+    )
+    @mock.patch("azure.monitor.opentelemetry.exporter.statsbeat._statsbeat.collect_statsbeat_metrics")
+    def test_transmit_response_error_statsbeat(self, stats_mock):
+        exporter = BaseExporter(disable_offline_storage=True)
+        with mock.patch.object(AzureMonitorClient, "track", throw(ServiceResponseError, message="Read timed out")):
+            result = exporter._transmit(self._envelopes_to_export)
+        stats_mock.assert_called_once()
+        self.assertEqual(len(_REQUESTS_MAP), 3)
+        self.assertEqual(_REQUESTS_MAP[_REQ_EXCEPTION_NAME[1]]["ServiceResponseError"], 1)
+        self.assertIsNotNone(_REQUESTS_MAP[_REQ_DURATION_NAME[1]])
+        self.assertEqual(_REQUESTS_MAP["count"], 1)
+        self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
+
     def test_transmit_request_exception(self):
         with mock.patch.object(AzureMonitorClient, "track", throw(Exception)):
             result = self._base._transmit(self._envelopes_to_export)