Skip to content

Commit 51f64b7

Browse files
andrewkrohandrewkroh
committed
[aws] Add var_groups for credential type selection with Identity Federation
Bump format_version to 3.6.0 and version to 7.0.0. Reorganize AWS credential configuration into a `Setup Access` var_groups selector with six options: Identity Federation, Direct Access Keys, Temporary Access Keys, Assume Role, Assume Role with External ID, and Shared Credentials. Key changes: - format_version: 3.4.0 → 3.6.0 - version: 6.20.3 → 7.0.0 - kibana.version: "^8.19.4 || ^9.2.1" → "^9.4.0" - agent.version: "^9.4.0" - var_groups: credential_type selector with 6 options - external_id is now secret: true - New vars: assume_role_duration, assume_role_expiry_window, supports_cloud_connectors - hide_in_var_group_options for 13 inputs across services that don't support Identity Federation - GuardDuty httpjson stream: switch to auth.aws: block and add Identity Federation policy tests - Add conditions.agent.ver^9.4.0 because guardduty now requires it. Source: #19278 (Omolola-Akinleye/integrations)
1 parent 910a126 commit 51f64b7

7 files changed

Lines changed: 366 additions & 20 deletions

File tree

‎packages/aws/changelog.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,21 @@
11
# newer versions go on top
2+
- version: "7.0.0"
3+
changes:
4+
- description: Reorganize AWS credentials configuration into a `Setup Access` selector with Identity Federation, Direct Access Keys, Temporary Access Keys, Assume Role, Assume Role with External ID, and Shared Credentials options. Existing access keys, role ARN, and shared credential settings continue to function and are auto-mapped to the corresponding option.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/19278
7+
- description: Switch the GuardDuty httpjson stream from manual SigV4 HMAC signing to the native `auth.aws` block.
8+
type: enhancement
9+
link: https://github.com/elastic/integrations/pull/19278
10+
- description: Store the `external_id` variable as a secret.
11+
type: breaking-change
12+
link: https://github.com/elastic/integrations/pull/19278
13+
- description: Add `assume_role_duration`, `assume_role_expiry_window`, and `supports_cloud_connectors` variables.
14+
type: enhancement
15+
link: https://github.com/elastic/integrations/pull/19278
16+
- description: Require Kibana and Elastic Agent ^9.4.0 (drop support for Kibana 8.x, Kibana 9.x below 9.4.0, and Elastic Agent below 9.4.0). The var_groups manifest feature and `auth.aws` runtime require this minimum.
17+
type: breaking-change
18+
link: https://github.com/elastic/integrations/pull/19278
219
- version: "6.20.3"
320
changes:
421
- description: Add processor tags to all ingest pipeline processors to satisfy elastic-package linter requirements for format_version 3.6.0 and above.

‎packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-agentless-cloud-connector.expected‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
inputs:
2+
- data_stream:
3+
namespace: ep
4+
meta:
5+
package:
6+
name: aws
7+
name: test-httpjson-agentless-cloud-connector-aws
8+
streams:
9+
- auth.aws:
10+
external_id: ${SECRET_0}
11+
role_arn: arn:aws:iam::123456789012:role/ElasticGuardDutyReadOnly
12+
use_cloud_connectors: true
13+
chain:
14+
- step:
15+
replace: $.nextToken
16+
request.method: POST
17+
request.ssl: null
18+
request.timeout: 30s
19+
request.transforms:
20+
- set:
21+
target: body.findingIds
22+
value: '[[toJSON .parent_last_response.body.findingIds]]'
23+
value_type: json
24+
- set:
25+
target: body.sortCriteria
26+
value: '{"attributeName":"updatedAt","orderBy":"ASC"}'
27+
value_type: json
28+
request.url: https://guardduty.us-east-1.amazonaws.com/detector/12abc34d567e8fa901bc2d34e567f890/findings/get
29+
response.split:
30+
target: body.findings
31+
config_version: 2
32+
cursor:
33+
last_execution_datetime:
34+
ignore_empty_value: true
35+
value: '[[$f := (index .last_response.body "findings")]][[if $f]][[if (ne (len $f) 50)]][[.last_event.updatedAt]][[end]][[end]]'
36+
data_stream:
37+
dataset: aws.guardduty
38+
interval: 5m
39+
publisher_pipeline.disable_host: true
40+
request.method: POST
41+
request.ssl: null
42+
request.timeout: 30s
43+
request.transforms:
44+
- set:
45+
target: body.maxResults
46+
value: 50
47+
value_type: int
48+
- set:
49+
target: body.sortCriteria
50+
value: '{"attributeName":"updatedAt","orderBy":"ASC"}'
51+
value_type: json
52+
- set:
53+
default: '[[((now (parseDuration "-48h"))).UnixMilli]]'
54+
target: body.findingCriteria.criterion.updatedAt.greaterThan
55+
value: '[[((parseDate .cursor.last_execution_datetime)).UnixMilli]]'
56+
request.url: https://guardduty.us-east-1.amazonaws.com/detector/12abc34d567e8fa901bc2d34e567f890/findings
57+
response.pagination:
58+
- set:
59+
do_not_log_failure: true
60+
fail_on_template_error: true
61+
target: body.nextToken
62+
value: '[[if (ne .last_response.body.nextToken "")]][[.last_response.body.nextToken]][[end]]'
63+
tags:
64+
- forwarded
65+
- aws-guardduty
66+
type: httpjson
67+
use_output: default
68+
output_permissions:
69+
default:
70+
_elastic_agent_checks:
71+
cluster:
72+
- monitor
73+
_elastic_agent_monitoring:
74+
indices: []
75+
uuid-for-permissions-on-related-indices:
76+
indices:
77+
- names:
78+
- logs-aws.guardduty-ep
79+
privileges:
80+
- auto_configure
81+
- create_doc
82+
secret_references:
83+
- {}

‎packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-agentless-cloud-connector.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
vars:
2+
role_arn: arn:aws:iam::123456789012:role/ElasticGuardDutyReadOnly
3+
external_id: guardduty-external-id
4+
supports_cloud_connectors: true
5+
default_region: us-east-1
6+
data_stream:
7+
vars:
8+
interval: 5m
9+
initial_interval: 48h
10+
detector_id: 12abc34d567e8fa901bc2d34e567f890
11+
aws_region: us-east-1
12+
tld: amazonaws.com
13+
http_client_timeout: 30s
14+
preserve_original_event: false
15+
preserve_duplicate_custom_fields: false

‎packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-legacy-credentials.expected‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
inputs:
2+
- data_stream:
3+
namespace: ep
4+
meta:
5+
package:
6+
name: aws
7+
name: test-httpjson-legacy-credentials-aws
8+
streams:
9+
- auth.aws:
10+
access_key_id: ${SECRET_0}
11+
secret_access_key: ${SECRET_1}
12+
chain:
13+
- step:
14+
replace: $.nextToken
15+
request.method: POST
16+
request.proxy_url: https://user:P%40ssword%23@192.0.2.10:8080
17+
request.ssl:
18+
enabled: true
19+
verification_mode: none
20+
request.timeout: 30s
21+
request.transforms:
22+
- set:
23+
target: body.findingIds
24+
value: '[[toJSON .parent_last_response.body.findingIds]]'
25+
value_type: json
26+
- set:
27+
target: body.sortCriteria
28+
value: '{"attributeName":"updatedAt","orderBy":"ASC"}'
29+
value_type: json
30+
request.url: https://guardduty.us-east-1.amazonaws.com/detector/12abc34d567e8fa901bc2d34e567f890/findings/get
31+
response.split:
32+
target: body.findings
33+
config_version: 2
34+
cursor:
35+
last_execution_datetime:
36+
ignore_empty_value: true
37+
value: '[[$f := (index .last_response.body "findings")]][[if $f]][[if (ne (len $f) 50)]][[.last_event.updatedAt]][[end]][[end]]'
38+
data_stream:
39+
dataset: aws.guardduty
40+
interval: 5m
41+
processors:
42+
- add_fields:
43+
fields:
44+
env: test
45+
name: guardduty
46+
target: project
47+
publisher_pipeline.disable_host: true
48+
request.method: POST
49+
request.proxy_url: https://user:P%40ssword%23@192.0.2.10:8080
50+
request.ssl:
51+
enabled: true
52+
verification_mode: none
53+
request.timeout: 30s
54+
request.tracer.filename: ../../logs/httpjson/http-request-trace-*.ndjson
55+
request.tracer.maxbackups: 5
56+
request.transforms:
57+
- set:
58+
target: body.maxResults
59+
value: 50
60+
value_type: int
61+
- set:
62+
target: body.sortCriteria
63+
value: '{"attributeName":"updatedAt","orderBy":"ASC"}'
64+
value_type: json
65+
- set:
66+
default: '[[((now (parseDuration "-48h"))).UnixMilli]]'
67+
target: body.findingCriteria.criterion.updatedAt.greaterThan
68+
value: '[[((parseDate .cursor.last_execution_datetime)).UnixMilli]]'
69+
request.url: https://guardduty.us-east-1.amazonaws.com/detector/12abc34d567e8fa901bc2d34e567f890/findings
70+
response.pagination:
71+
- set:
72+
do_not_log_failure: true
73+
fail_on_template_error: true
74+
target: body.nextToken
75+
value: '[[if (ne .last_response.body.nextToken "")]][[.last_response.body.nextToken]][[end]]'
76+
tags:
77+
- preserve_original_event
78+
- preserve_duplicate_custom_fields
79+
- forwarded
80+
- aws-guardduty
81+
- test-policy
82+
type: httpjson
83+
use_output: default
84+
output_permissions:
85+
default:
86+
_elastic_agent_checks:
87+
cluster:
88+
- monitor
89+
_elastic_agent_monitoring:
90+
indices: []
91+
uuid-for-permissions-on-related-indices:
92+
indices:
93+
- names:
94+
- logs-aws.guardduty-ep
95+
privileges:
96+
- auto_configure
97+
- create_doc
98+
secret_references:
99+
- {}
100+
- {}

‎packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-legacy-credentials.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
vars:
2+
access_key_id: FAKE_AWS_ACCESS_KEY_ID_FOR_TESTS_ONLY
3+
secret_access_key: FAKE_AWS_SECRET_ACCESS_KEY_FOR_TESTS_ONLY
4+
default_region: us-east-1
5+
data_stream:
6+
vars:
7+
enable_request_tracer: true
8+
interval: 5m
9+
initial_interval: 48h
10+
detector_id: 12abc34d567e8fa901bc2d34e567f890
11+
aws_region: us-east-1
12+
tld: amazonaws.com
13+
http_client_timeout: 30s
14+
proxy_url: https://user:P%40ssword%23@192.0.2.10:8080
15+
ssl: |
16+
enabled: true
17+
verification_mode: none
18+
preserve_original_event: true
19+
preserve_duplicate_custom_fields: true
20+
tags:
21+
- forwarded
22+
- aws-guardduty
23+
- test-policy
24+
processors: |
25+
- add_fields:
26+
target: project
27+
fields:
28+
name: guardduty
29+
env: test

‎packages/aws/data_stream/guardduty/agent/stream/httpjson.yml.hbs‎

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,8 @@ request.proxy_url: {{proxy_url}}
1515
{{#if ssl}}
1616
request.ssl: {{ssl}}
1717
{{/if}}
18+
1819
request.transforms:
19-
- set:
20-
target: header.X-Amz-Date
21-
value: '[[formatDate (now) "20060102T150405Z"]]'
2220
- set:
2321
target: body.maxResults
2422
value: 50
@@ -35,20 +33,12 @@ request.transforms:
3533
re-evaluates (now) on every pagination page, which can change the query and
3634
invalidate the NextToken. A CEL rewrite can reintroduce the upper bound safely
3735
since it can pin the value once per collection interval. --}}
38-
- set:
39-
target: header.Authorization
40-
value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/guardduty/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "guardduty")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/guardduty/aws4_request") (hash "sha256" "POST\n" "/detector/{{detector_id}}/findings\n" "\n" "host:guardduty.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]'
4136
response.pagination:
4237
- set:
4338
target: body.nextToken
4439
value: '[[if (ne .last_response.body.nextToken "")]][[.last_response.body.nextToken]][[end]]'
4540
fail_on_template_error: true
4641
do_not_log_failure: true
47-
- delete:
48-
target: header.Authorization
49-
- set:
50-
target: header.Authorization
51-
value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/guardduty/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "guardduty")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/guardduty/aws4_request") (hash "sha256" "POST\n" "/detector/{{detector_id}}/findings\n" "\n" "host:guardduty.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]'
5242
chain:
5343
- step:
5444
request.url: https://guardduty.{{aws_region}}.{{tld}}/detector/{{detector_id}}/findings/get
@@ -64,9 +54,6 @@ chain:
6454
{{/if}}
6555
request.method: POST
6656
request.transforms:
67-
- set:
68-
target: header.X-Amz-Date
69-
value: '[[formatDate (now) "20060102T150405Z"]]'
7057
- set:
7158
target: body.findingIds
7259
value: '[[toJSON .parent_last_response.body.findingIds]]'
@@ -75,9 +62,6 @@ chain:
7562
target: body.sortCriteria
7663
value: '{"attributeName":"updatedAt","orderBy":"ASC"}'
7764
value_type: json
78-
- set:
79-
target: header.Authorization
80-
value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/guardduty/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "guardduty")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/guardduty/aws4_request") (hash "sha256" "POST\n" "/detector/{{detector_id}}/findings/get\n" "\n" "host:guardduty.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]'
8165
response.split:
8266
target: body.findings
8367
cursor:
@@ -101,3 +85,34 @@ publisher_pipeline.disable_host: true
10185
processors:
10286
{{processors}}
10387
{{/if}}
88+
auth.aws:
89+
{{#if access_key_id}}
90+
access_key_id: {{access_key_id}}
91+
{{/if}}
92+
{{#if secret_access_key}}
93+
secret_access_key: {{secret_access_key}}
94+
{{/if}}
95+
{{#if session_token}}
96+
session_token: {{session_token}}
97+
{{/if}}
98+
{{#if shared_credential_file}}
99+
shared_credential_file: {{shared_credential_file}}
100+
{{/if}}
101+
{{#if credential_profile_name}}
102+
credential_profile_name: {{credential_profile_name}}
103+
{{/if}}
104+
{{#if role_arn}}
105+
role_arn: {{role_arn}}
106+
{{/if}}
107+
{{#if external_id}}
108+
external_id: {{external_id}}
109+
{{/if}}
110+
{{#if assume_role_duration}}
111+
assume_role.duration: {{assume_role_duration}}
112+
{{/if}}
113+
{{#if assume_role_expiry_window}}
114+
assume_role.expiry_window: {{assume_role_expiry_window}}
115+
{{/if}}
116+
{{#if supports_cloud_connectors}}
117+
use_cloud_connectors: {{supports_cloud_connectors}}
118+
{{/if}}

0 commit comments

Comments
 (0)