Send errors metrics for 5xx response from API Gateway, Lambda Function URL, or ALB (#229)

Author: kimi-p

datadog_lambda/tracing.py

@@ -10,6 +10,8 @@
 from datetime import datetime, timezone
 from typing import Optional, Dict
 
+from datadog_lambda.metric import submit_errors_metric
+
 try:
     from typing import Literal
 except ImportError:
@@ -959,6 +961,13 @@ def create_function_execution_span(
     return span
 
 
+def mark_trace_as_error_for_5xx_responses(context, status_code, span):
+    if len(status_code) == 3 and status_code.startswith("5"):
+        submit_errors_metric(context)
+        if span:
+            span.error = 1
+
+
 class InferredSpanInfo(object):
     BASE_NAME = "_inferred_span"
     SYNCHRONICITY = f"{BASE_NAME}.synchronicity"

datadog_lambda/wrapper.py

@@ -11,8 +11,8 @@
 from datadog_lambda.extension import should_use_extension, flush_extension
 from datadog_lambda.cold_start import set_cold_start, is_cold_start
 from datadog_lambda.constants import (
-    XraySubsegment,
     TraceContextSource,
+    XraySubsegment,
 )
 from datadog_lambda.metric import (
     flush_stats,
@@ -26,6 +26,7 @@
     create_dd_dummy_metadata_subsegment,
     inject_correlation_ids,
     dd_tracing_enabled,
+    mark_trace_as_error_for_5xx_responses,
     set_correlation_ids,
     set_dd_trace_py_root,
     create_function_execution_span,
@@ -151,7 +152,7 @@ def __call__(self, event, context, **kwargs):
 
     def _before(self, event, context):
         try:
-
+            self.response = None
             set_cold_start()
             submit_invocations_metric(context)
             self.trigger_tags = extract_trigger_tags(event, context)
@@ -190,6 +191,8 @@ def _after(self, event, context):
             status_code = extract_http_status_code_tag(self.trigger_tags, self.response)
             if status_code:
                 self.trigger_tags["http.status_code"] = status_code
+                mark_trace_as_error_for_5xx_responses(context, status_code, self.span)
+
             # Create a new dummy Datadog subsegment for function trigger tags so we
             # can attach them to X-Ray spans when hybrid tracing is used
             if self.trigger_tags:

tests/test_tracing.py

@@ -2,8 +2,10 @@
 import json
 import os
 
-from unittest.mock import MagicMock, patch, call
+from unittest.mock import MagicMock, Mock, patch, call
 
+import ddtrace
+from ddtrace.constants import ERROR_MSG, ERROR_TYPE
 from ddtrace.helpers import get_correlation_ids
 from ddtrace.context import Context
 
@@ -18,6 +20,7 @@
     create_dd_dummy_metadata_subsegment,
     create_function_execution_span,
     get_dd_trace_context,
+    mark_trace_as_error_for_5xx_responses,
     set_correlation_ids,
     set_dd_trace_py_root,
     _convert_xray_trace_id,
@@ -1191,3 +1194,24 @@ def test_create_inferred_span_from_api_gateway_event_no_apiid(self):
         self.assertEqual(span.span_type, "http")
         self.assertEqual(span.get_tag(InferredSpanInfo.TAG_SOURCE), "self")
         self.assertEqual(span.get_tag(InferredSpanInfo.SYNCHRONICITY), "sync")
+
+    @patch("datadog_lambda.tracing.submit_errors_metric")
+    def test_mark_trace_as_error_for_5xx_responses_getting_400_response_code(
+        self, mock_submit_errors_metric
+    ):
+        mark_trace_as_error_for_5xx_responses(
+            context="fake_context", status_code="400", span="empty_span"
+        )
+        mock_submit_errors_metric.assert_not_called()
+
+    @patch("datadog_lambda.tracing.submit_errors_metric")
+    def test_mark_trace_as_error_for_5xx_responses_sends_error_metric_and_set_error_tags(
+        self, mock_submit_errors_metric
+    ):
+        mock_span = Mock(ddtrace.span.Span)
+        status_code = "500"
+        mark_trace_as_error_for_5xx_responses(
+            context="fake_context", status_code=status_code, span=mock_span
+        )
+        mock_submit_errors_metric.assert_called_once()
+        self.assertEqual(1, mock_span.error)

tests/test_wrapper.py

@@ -275,6 +275,61 @@ def lambda_handler(event, context):
             ]
         )
 
+    @patch("datadog_lambda.wrapper.extract_trigger_tags")
+    def test_5xx_sends_errors_metric_and_set_tags(self, mock_extract_trigger_tags):
+        mock_extract_trigger_tags.return_value = {
+            "function_trigger.event_source": "api-gateway",
+            "function_trigger.event_source_arn": "arn:aws:apigateway:us-west-1::/restapis/1234567890/stages/prod",
+            "http.url": "70ixmpl4fl.execute-api.us-east-2.amazonaws.com",
+            "http.url_details.path": "/prod/path/to/resource",
+            "http.method": "GET",
+        }
+
+        @datadog_lambda_wrapper
+        def lambda_handler(event, context):
+            return {"statusCode": 500, "body": "fake response body"}
+
+        lambda_event = {}
+
+        lambda_handler(lambda_event, get_mock_context())
+
+        self.mock_write_metric_point_to_stdout.assert_has_calls(
+            [
+                call(
+                    "aws.lambda.enhanced.invocations",
+                    1,
+                    tags=[
+                        "region:us-west-1",
+                        "account_id:123457598159",
+                        "functionname:python-layer-test",
+                        "resource:python-layer-test:1",
+                        "cold_start:true",
+                        "memorysize:256",
+                        "runtime:python3.9",
+                        "datadog_lambda:v6.6.6",
+                        "dd_lambda_layer:datadog-python39_X.X.X",
+                    ],
+                    timestamp=None,
+                ),
+                call(
+                    "aws.lambda.enhanced.errors",
+                    1,
+                    tags=[
+                        "region:us-west-1",
+                        "account_id:123457598159",
+                        "functionname:python-layer-test",
+                        "resource:python-layer-test:1",
+                        "cold_start:true",
+                        "memorysize:256",
+                        "runtime:python3.9",
+                        "datadog_lambda:v6.6.6",
+                        "dd_lambda_layer:datadog-python39_X.X.X",
+                    ],
+                    timestamp=None,
+                ),
+            ]
+        )
+
     def test_enhanced_metrics_cold_start_tag(self):
         @datadog_lambda_wrapper
         def lambda_handler(event, context):