feat(aws): AWSX-1566 Adding storage tag HTTP header

Signed-off-by: Vincent Boutour <vincent.boutour@datadoghq.com>

Author: ViBiOh

aws/logs_monitoring/README.md

@@ -468,6 +468,7 @@ For all configuration options and details, including [Multi-Region deployment][2
 [203]: https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site
 [204]: https://app.datadoghq.com/organization-settings/api-keys
 [205]: https://registry.terraform.io/modules/DataDog/log-lambda-forwarder-datadog/aws/latest#multi-region-deployments
+
 {{% /tab %}}
 {{% tab "Manual" %}}
 
@@ -567,11 +568,17 @@ To test different patterns against your logs, turn on [debug logs](#troubleshoot
 
 ### Advanced (optional)
 
+`DD_ENRICH_S3_TAGS`
+: Instruct Datadog Backend to enrich a log coming from a S3 bucket with the tag attached to this bucket. It's the equivalent behavior of `DD_FETCH_S3_TAG` but done after ingestion. This require Resource Collection to be enabled. Enabled by default.
+
+`DD_ENRICH_CLOUDWATCH_TAGS`
+: Instruct Datadog Backend to enrich a log coming from a Cloudwatch logGroup with the tag attached to this logGroup. It's the equivalent behavior of `DD_FETCH_LOG_GROUP_TAGS` but done after ingestion. This require Resource Collection to be enabled.
+
 `DD_FETCH_LAMBDA_TAGS`
 : Let the Forwarder fetch Lambda tags using GetResources API calls and apply them to logs, metrics, and traces. If set to true, permission `tag:GetResources` will be automatically added to the Lambda execution IAM role.
 
 `DD_FETCH_LOG_GROUP_TAGS`
-: Let the forwarder fetch Log Group tags using ListTagsLogGroup and apply them to logs, metrics, and traces. If set to true, permission `logs:ListTagsForResource` will be automatically added to the Lambda execution IAM role.
+: [DEPRECATED, use DD_ENRICH_CLOUDWATCH_TAG] Let the forwarder fetch Log Group tags using ListTagsLogGroup and apply them to logs, metrics, and traces. If set to true, permission `logs:ListTagsForResource` will be automatically added to the Lambda execution IAM role.
 
 `DD_FETCH_STEP_FUNCTIONS_TAGS`
 : Let the Forwarder fetch Step Functions tags using GetResources API calls and apply them to logs and traces (if Step Functions tracing is enabled). If set to true, permission `tag:GetResources` will be automatically added to the Lambda execution IAM role.

aws/logs_monitoring/logs/datadog_http_client.py

@@ -4,19 +4,21 @@
 # Copyright 2021 Datadog, Inc.
 
 
-import os
 import logging
-
+import os
 from concurrent.futures import as_completed
+
 from requests_futures.sessions import FuturesSession
-from logs.helpers import compress_logs
-from logs.exceptions import ScrubbingException
 
+from logs.exceptions import ScrubbingException
+from logs.helpers import compress_logs
 from settings import (
-    DD_USE_COMPRESSION,
     DD_COMPRESSION_LEVEL,
-    DD_MAX_WORKERS,
     DD_FORWARDER_VERSION,
+    DD_MAX_WORKERS,
+    DD_USE_COMPRESSION,
+    get_enrich_cloudwatch_tags,
+    get_enrich_s3_tags,
 )
 
 logger = logging.getLogger()
@@ -37,6 +39,10 @@ class DatadogHTTPClient(object):
     _HEADERS["DD-EVP-ORIGIN"] = "aws_forwarder"
     _HEADERS["DD-EVP-ORIGIN-VERSION"] = DD_FORWARDER_VERSION
 
+    storage_tag = get_dd_storage_tag_header()
+    if storage_tag != "":
+        _HEADERS["DD-STORAGE-TAG"] = storage_tag
+
     def __init__(
         self, host, port, no_ssl, skip_ssl_validation, api_key, scrubber, timeout=10
     ):
@@ -92,3 +98,17 @@ def __enter__(self):
 
     def __exit__(self, ex_type, ex_value, traceback):
         self._close()
+
+
+def get_dd_storage_tag_header():
+    storage_tag = ""
+
+    if get_enrich_s3_tags():
+        storage_tag += "s3"
+
+    if get_enrich_cloudwatch_tags():
+        if storage_tag != "":
+            storage_tag += ","
+        storage_tag += "cloudwatch"
+
+    return storage_tag

aws/logs_monitoring/settings.py

@@ -250,6 +250,13 @@ def __init__(self, name, pattern, placeholder, enabled=True):
 )
 
 
+DD_ENRICH_S3_TAGS = get_env_var("DD_ENRICH_S3_TAGS", default="true", boolean=True)
+
+DD_ENRICH_CLOUDWATCH_TAGS = get_env_var(
+    "DD_ENRICH_CLOUDWATCH_TAGS", default="false", boolean=True
+)
+
+
 def get_fetch_s3_tags():
     return DD_FETCH_S3_TAGS
 
@@ -266,6 +273,14 @@ def get_fetch_step_functions_tags():
     return DD_FETCH_STEP_FUNCTIONS_TAGS
 
 
+def get_enrich_s3_tags():
+    return DD_ENRICH_S3_TAGS
+
+
+def get_enrich_cloudwatch_tags():
+    return DD_ENRICH_CLOUDWATCH_TAGS
+
+
 DD_SOURCE = "ddsource"
 DD_CUSTOM_TAGS = "ddtags"
 DD_SERVICE = "service"