[aws/config]: Skip service-linked rules and tolerate per-rule errors (#19803)

ShourieG · web-flow · commit 5fa62a7cf2d8 · 2026-06-26T23:43:06.000+05:30
aws/config: skip service-linked rules and tolerate per-rule errors

The AWS Config data stream listed every Config rule and then called
GetComplianceDetailsByConfigRule for each one. AWS owns service-linked
rules and blocks that call for them with an AccessDeniedException, which
aborted the whole collection cycle and degraded the input so no rules
were ingested.

Filter out service-linked rules in the DescribeConfigRules response by
dropping any rule with a populated CreatedBy field, which AWS sets only
for service-linked rules, so the blocked call is never attempted. As a
defence-in-depth measure, treat a per-rule GetComplianceDetailsByConfigRule
failure as non-fatal: emit the error as an advancing event and continue
with the remaining rules instead of stopping the cycle. Thread the
resource url through every output state branch so the program stays
self-contained across evaluations.

Add a service-linked rule and a matching AccessDeniedException fixture to
the system-test mock to confirm the rule is filtered and never queried,
and bump the package to 6.20.2.
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "6.20.2"
+  changes:
+    - description: Skip AWS service-linked Config rules when collecting compliance details so the AWS Config data stream no longer degrades on `AccessDeniedException`, and treat per-rule compliance errors as non-fatal so a single failing rule does not stop ingestion of the rest.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/19803
 - version: "6.20.1"
   changes:
     - description: Revert addition of assignment of host.id from CloudTrail target hosts.
diff --git a/packages/aws/data_stream/config/_dev/deploy/docker/files/config.yml b/packages/aws/data_stream/config/_dev/deploy/docker/files/config.yml
@@ -88,6 +88,23 @@ rules:
                   "SourceIdentifier": "ACCESS_KEYS_ROTATED"
                 }
               },
+              {
+                "ConfigRuleArn": "arn:aws:config:us-east-1:11223344556:config-rule/config-rule-slr",
+                "ConfigRuleId": "config-rule-slr",
+                "ConfigRuleName": "service-linked-config-rule",
+                "ConfigRuleState": "ACTIVE",
+                "CreatedBy": "securityhub.amazonaws.com",
+                "Description": "AWS service-linked rule owned by AWS Security Hub. Querying its compliance details is blocked by AWS with AccessDeniedException, so the integration must skip it.",
+                "EvaluationModes": [
+                  {
+                    "Mode": "DETECTIVE"
+                  }
+                ],
+                "Source": {
+                  "Owner": "AWS",
+                  "SourceIdentifier": "SECURITYHUB_SERVICE_LINKED"
+                }
+              },
               {
                 "ConfigRuleArn": "arn:aws:config:us-east-1:11223344556:config-rule/config-rule-id2",
                 "ConfigRuleId": "config-rule-id2",
@@ -234,3 +251,25 @@ rules:
             ]
           }
           `}}
+  # Safety net: AWS blocks GetComplianceDetailsByConfigRule for service-linked
+  # rules. The integration must skip them via the CreatedBy filter and never
+  # reach this response. If the filter regresses and this rule is queried, AWS
+  # returns this 400 AccessDeniedException, which the program must treat as
+  # non-fatal (advance past it) rather than aborting the whole collection cycle.
+  - path: /
+    methods: ["POST"]
+    request_headers:
+      Content-Type:
+        - "application/x-amz-json-1.1"
+      X-Amz-Target:
+        - "StarlingDoveService.GetComplianceDetailsByConfigRule"
+    request_body: '{"ConfigRuleName":"service-linked-config-rule","Limit":2}'
+    responses:
+      - status_code: 400
+        body: |-
+          {{ minify_json `
+          {
+            "__type": "AccessDeniedException",
+            "Message": "An AWS service owns ServiceLinkedConfigRule. You do not have permissions to take action on this rule."
+          }
+          `}}
diff --git a/packages/aws/data_stream/config/agent/stream/cel.yml.hbs b/packages/aws/data_stream/config/agent/stream/cel.yml.hbs
@@ -65,6 +65,7 @@ program: |
       {
         // State management: preserve existing state from previous iteration
         "worklist": state.worklist,
+        "url": state.url,
         "batch_size": state.batch_size,
         "has_next": state.has_next,
         "next": state.next,
@@ -188,10 +189,21 @@ program: |
           }
         ).do_request().as(resp, (resp.StatusCode == 200) ?
           // Response Processing: Successfully got config rules list
-          resp.Body.decode_json().as(body,
+          resp.Body.decode_json().as(raw_body,
+            // Skip AWS service-linked rules. AWS owns these rules and blocks
+            // GetComplianceDetailsByConfigRule for them with an AccessDeniedException
+            // ("An AWS service owns ServiceLinkedConfigRule"). They are identified by a
+            // populated CreatedBy field, which AWS sets only for service-linked rules.
+            // Filtering them out of the worklist here avoids the blocked call entirely.
+            raw_body.with({
+              "ConfigRules": raw_body.?ConfigRules.orValue([]).filter(r,
+                r.?CreatedBy.orValue("") == ""
+              ),
+            }).as(body,
             {
               // State Update: Store the fetched config rules as worklist
               "worklist": body,
+              "url": state.url,
               "next": 0,
               "has_next": has(body.NextToken),
               "batch_size": state.batch_size,
@@ -238,11 +250,12 @@ program: |
                 ].join("\n").sha256().hex()
               ].join("\n"),
               ?"session_token": state.?session_token,
+              "url": state.url,
               "access_key": state.access_key,
               "secret_key": state.secret_key,
               "aws_region": state.aws_region,
               "tld": state.tld,
-            }
+            })
           )
         :
           // Error Handling: DescribeConfigRules request failed
@@ -263,6 +276,7 @@ program: |
             "want_more": false,
             "batch_size": state.batch_size,
             ?"session_token": state.?session_token,
+            "url": state.url,
             "access_key": state.access_key,
             "secret_key": state.secret_key,
             "aws_region": state.aws_region,
@@ -344,49 +358,95 @@ program: |
             "has_next": config_rules.has_next,
             "batch_size": config_rules.batch_size,
             ?"session_token": config_rules.?session_token,
+            "url": config_rules.url,
             "access_key": config_rules.access_key,
             "secret_key": config_rules.secret_key,
             "aws_region": config_rules.aws_region,
             "tld": config_rules.tld,
           }
         )
       :
-        // Error Handling: GetComplianceDetailsByConfigRule request failed
+        // Error Handling: GetComplianceDetailsByConfigRule request failed for this
+        // rule. Treat it as non-fatal: emit the error as an advancing (array) event
+        // and continue with the remaining rules instead of aborting the whole cycle.
+        // This prevents a single per-rule access error (for example a service-linked
+        // rule that slipped past the CreatedBy filter) from degrading the entire input
+        // and stopping all ingestion. The array form advances the cursor; the
+        // ingest pipeline's terminate processor handles the error event.
         {
-          "events": {
-            "error": {
-              "code": string(resp.StatusCode),
-              "id": string(resp.Status),
-              "message": "GetComplianceDetailsByConfigRule: POST " + state.url.trim_right("/") + " " +
-              (
-                (size(resp.Body) != 0) ?
-                  string(resp.Body)
-                :
-                  string(resp.Status) + " (" + string(resp.StatusCode) + ")"
-              ),
+          "events": [
+            {
+              "error": {
+                "code": string(resp.StatusCode),
+                "id": string(resp.Status),
+                "message": "GetComplianceDetailsByConfigRule: POST " + state.url.trim_right("/") + " " +
+                (
+                  (size(resp.Body) != 0) ?
+                    string(resp.Body)
+                  :
+                    string(resp.Status) + " (" + string(resp.StatusCode) + ")"
+                ),
+              },
             },
+          ],
+          "next_page": {
+            ?"rule_token": config_rules.?next_page.rule_token,
           },
-          "want_more": false,
+          "want_more": config_rules.has_next || (int(config_rules.next) + 1 < size(config_rules.worklist.ConfigRules)),
+          "next": (int(config_rules.next) + 1 < size(config_rules.worklist.ConfigRules)) ?
+            (int(config_rules.next) + 1)
+          :
+            0,
+          "worklist": (int(config_rules.next) + 1 < size(config_rules.worklist.ConfigRules)) ?
+            config_rules.worklist
+          :
+            {},
+          "has_next": config_rules.has_next,
           "batch_size": config_rules.batch_size,
           ?"session_token": config_rules.?session_token,
+          "url": config_rules.url,
           "access_key": config_rules.access_key,
           "secret_key": config_rules.secret_key,
           "aws_region": config_rules.aws_region,
           "tld": config_rules.tld,
         }
       )
     :
-      // Final State: No config rules found, return empty result
-      {
-        "events": [],
-        "want_more": false,
-        "batch_size": config_rules.batch_size,
-        ?"session_token": config_rules.?session_token,
-        "access_key": config_rules.access_key,
-        "secret_key": config_rules.secret_key,
-        "aws_region": config_rules.aws_region,
-        "tld": config_rules.tld,
-      }
+      // Final State: No config rules to process in this page. If DescribeConfigRules
+      // paginated (rule_token present) or another rule page is pending (has_next) —
+      // for example because every rule on this page was a filtered service-linked
+      // rule — emit a placeholder event so the cursor persists and continue fetching;
+      // otherwise return an empty result and stop.
+      (config_rules.?has_next.orValue(false) || has(config_rules.?next_page.rule_token)) ?
+        {
+          "events": [{"message": "retry"}],
+          "next_page": {
+            ?"rule_token": config_rules.?next_page.rule_token,
+          },
+          "want_more": true,
+          "next": 0,
+          "worklist": {},
+          "has_next": config_rules.?has_next.orValue(false),
+          "batch_size": config_rules.batch_size,
+          ?"session_token": config_rules.?session_token,
+          "url": config_rules.url,
+          "access_key": config_rules.access_key,
+          "secret_key": config_rules.secret_key,
+          "aws_region": config_rules.aws_region,
+          "tld": config_rules.tld,
+        }
+      :
+        {
+          "events": [],
+          "want_more": false,
+          "batch_size": config_rules.batch_size,
+          ?"session_token": config_rules.?session_token,
+          "url": config_rules.url,
+          "access_key": config_rules.access_key,
+          "secret_key": config_rules.secret_key,
+          "aws_region": config_rules.aws_region,
+          "tld": config_rules.tld,
+        }
   )
 tags:
 {{#if preserve_original_event}}
diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.4.0
 name: aws
 title: AWS
-version: 6.20.1
+version: 6.20.2
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:
