[aws] Add processor tags to ingest pipelines (#19824)

Add `tag` fields to every processor in all AWS ingest pipelines using
`elastic-package modify --modifiers pipeline-tag`. Also update
`on_failure` error messages to include `_ingest.pipeline` per the
SVR00009 linter rule. Both are requirements that will be enforced when
the AWS package format_version is bumped to 3.6.0.

Also fix two pre-existing duplicate processors exposed by the tagging:
- cloudfront_logs/default.yml: duplicate x-edge-response-result-type rename
- rds/default.yml: duplicate FreeStorageSpace rename

Fixes cloudfront_logs/pipeline_process_ip.yml which was missing a
pipeline-level on_failure block (SVR00008/SVR00009).

Author: andrewkroh

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "6.20.3"
+  changes:
+    - description: Add processor tags to all ingest pipeline processors to satisfy elastic-package linter requirements for format_version 3.6.0 and above.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/19824
 - version: "6.20.2"
   changes:
     - description: Skip AWS service-linked Config rules when collecting compliance details so the AWS Config data stream no longer degrades on `AccessDeniedException`, and treat per-rule compliance errors as non-fatal so a single failing rule does not stop ingestion of the rest.

packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml

@@ -3,85 +3,105 @@ description: "Pipeline for API Gateway logs in CloudWatch"
 
 processors:
   - set:
+      tag: set_ecs_version_0f7334bb
       field: ecs.version
       value: '8.11.0'
   - rename:
+      tag: rename_message_to_event_original_c74b1d7e
       field: message
       target_field: event.original
       ignore_missing: true
       if: 'ctx.event?.original == null'
       description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.'
   - remove:
+      tag: remove_message_2ee3a7d4
       field: message
       ignore_missing: true
       if: 'ctx.event?.original != null'
       description: 'The `message` field is no longer required if the document has an `event.original` field.'
   - json:
+      tag: json_event_original_to_aws_apigateway_f58e2e48
       field: event.original
       target_field: aws.apigateway
       ignore_failure: true
   - rename:
+      tag: rename_aws_apigateway_requestId_to_aws_apigateway_request_id_f8d45745
       field: aws.apigateway.requestId
       target_field: aws.apigateway.request_id
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_responseLength_to_aws_apigateway_response_length_351accf5
       field: aws.apigateway.responseLength
       target_field: aws.apigateway.response_length
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_requestTime_to_aws_apigateway_request_time_452b54dd
       field: aws.apigateway.requestTime
       target_field: aws.apigateway.request_time
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_httpMethod_to_aws_apigateway_http_method_0f8ceff7
       field: aws.apigateway.httpMethod
       target_field: aws.apigateway.http_method
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_routeKey_to_aws_apigateway_route_key_24f23c8b
       field: aws.apigateway.routeKey
       target_field: aws.apigateway.route_key
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_ip_to_aws_apigateway_ip_address_bcb43f2f
       field: aws.apigateway.ip
       target_field: aws.apigateway.ip_address
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_resourcePath_to_aws_apigateway_resource_path_a086dd8b
       field: aws.apigateway.resourcePath
       target_field: aws.apigateway.resource_path
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_connectionId_to_aws_apigateway_connection_id_6f806acf
       field: aws.apigateway.connectionId
       target_field: aws.apigateway.connection_id
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_eventType_to_aws_apigateway_event_type_2a463e2f
       field: aws.apigateway.eventType
       target_field: aws.apigateway.event_type
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_apiId_to_aws_apigateway_api_id_1b4980ad
       field: aws.apigateway.apiId
       target_field: aws.apigateway.api_id
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_domainName_to_aws_apigateway_domain_name_732b5853
       field: aws.apigateway.domainName
       target_field: aws.apigateway.domain_name
       ignore_missing: true
   - grok:
+      tag: grok_aws_apigateway_ip_address_e6f24967
       field: aws.apigateway.ip_address
       patterns:
         - '%{IPORHOST:aws.apigateway.ip_address}'
       ignore_failure: true
   - convert:
+      tag: convert_aws_apigateway_ip_address_10d88955
       field: aws.apigateway.ip_address
       type: ip
       ignore_missing: true
   - convert:
+      tag: convert_aws_apigateway_response_length_de377d20
       field: aws.apigateway.response_length
       type: long
       ignore_missing: true
   - convert:
+      tag: convert_aws_apigateway_status_28a91148
       field: aws.apigateway.status
       type: long
       ignore_missing: true
   - date:
+      tag: date_aws_apigateway_request_time_to_aws_apigateway_request_time_6995b9b1
       field: aws.apigateway.request_time
       target_field: "aws.apigateway.request_time"
       formats:
@@ -96,4 +116,4 @@ on_failure:
       value: >-
         Processor '{{{ _ingest.on_failure_processor_type }}}'
         {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
-        {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
+        {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'

packages/aws/data_stream/awshealth/elasticsearch/ingest_pipeline/default.yml

@@ -2,11 +2,13 @@
 description: Pipeline for AWS Health metrics
 processors:
 - script:
+    tag: script_aad5131d
     if: "ctx.aws != null && ctx.aws.awshealth != null && ctx.aws.awshealth.end_time == '0001-01-01T00:00:00.000Z'"
     "lang": "painless"
     "source": "ctx.aws.awshealth.end_time = null"
 
 - script:
+    tag: script_7a0bd239
     lang: painless
     source: |-
         boolean drop(Object o) {
@@ -33,4 +35,4 @@ on_failure:
     value: >-
       Processor '{{{ _ingest.on_failure_processor_type }}}'
       {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
-      {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
+      {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'

packages/aws/data_stream/billing/elasticsearch/ingest_pipeline/default.yml

@@ -2,14 +2,17 @@
 description: "Pipeline for AWS Billing"
 processors:
   - dot_expander:
+      tag: dot_expander_*_c2762c23
       field: "*"
       ignore_failure: true
   - set:
+      tag: set_cloud_account_name_20733914
       field: cloud.account.name
       copy_from: cloud.account.id
       override: false
       ignore_empty_value: true
   - fingerprint:
+      tag: fingerprint_deb4df05
       fields: ["aws.billing.group_by"]
       target_field: 'aws.billing.group_by.fingerprint'
       ignore_missing: true
@@ -22,4 +25,4 @@ on_failure:
       value: >-
         Processor '{{{ _ingest.on_failure_processor_type }}}'
         {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
-        {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
+        {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'