diff --git a/packages/sentinel_one/changelog.yml b/packages/sentinel_one/changelog.yml index cfa7ddf944e..9b20102d89a 100644 --- a/packages/sentinel_one/changelog.yml +++ b/packages/sentinel_one/changelog.yml @@ -1,4 +1,12 @@ # newer versions go on top +- version: "2.11.1" + changes: + - description: Add time watermark and fingerprint deduplication to the threat event data stream to prevent duplicate ingestion. + type: bugfix + link: https://github.com/elastic/integrations/pull/20211 + - description: Add fingerprint deduplication to the application risk data stream to prevent duplicate ingestion. + type: bugfix + link: https://github.com/elastic/integrations/pull/20211 - version: "2.11.0" changes: - description: | diff --git a/packages/sentinel_one/data_stream/application_risk/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/application_risk/elasticsearch/ingest_pipeline/default.yml index c1a92cd727e..f678357bcf6 100644 --- a/packages/sentinel_one/data_stream/application_risk/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one/data_stream/application_risk/elasticsearch/ingest_pipeline/default.yml @@ -26,6 +26,13 @@ processors: field: event.original tag: json_event_original target_field: json + - fingerprint: + tag: fingerprint_application_risk_dedup + fields: + - json.id + - json.status + target_field: _id + ignore_missing: true - set: field: event.kind tag: set_event_kind diff --git a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.expected b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.expected index aded9afd272..af6885d33e8 100644 --- a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.expected +++ b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.expected @@ -33,8 +33,17 @@ inputs: state.url.trim_right("/") + "/web/api/v2.1/threats?" + { "skipCount": ["true"], "limit": [string(state.batch_size)], + "sortBy": ["updatedAt"], + "sortOrder": ["asc"], + "updatedAt__gte": [ + state.?cursor.?list_updated_at_gte.orValue( + state.?cursor.?last_timestamp.orValue( + (now - duration(state.initial_interval)).format(time_layout.RFC3339) + ) + ) + ], ?"siteIds": state.?site_ids.optMap(v, [string(v)]), - ?"cursor": state.?cursor.next_page.token.optMap(v, [v]), + ?"cursor": state.?cursor.?next_page.token.optMap(v, [v]), }.format_query() ).with( { @@ -51,6 +60,12 @@ inputs: ?"token": (body.?pagination.nextCursor.orValue(null) != null) ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": body.?pagination.nextCursor.orValue(null) != null, + "list_updated_at_gte": state.?cursor.?list_updated_at_gte.orValue( + state.?cursor.?last_timestamp.orValue( + (now - duration(state.initial_interval)).format(time_layout.RFC3339) + ) + ), + ?"last_timestamp": state.?cursor.last_timestamp, }, } ) @@ -69,7 +84,6 @@ inputs: }, }, "want_more": false, - "cursor": {}, } ) ) @@ -119,6 +133,11 @@ inputs: ?"token": has_more_events ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": state.?cursor.fetch_more.orValue(false), + "list_updated_at_gte": state.cursor.list_updated_at_gte, + ?"last_timestamp": has_more_events ? + state.?cursor.last_timestamp + : + state.cursor.worklist.data[0].?threatInfo.updatedAt.or(state.?cursor.last_timestamp), }, } ) @@ -145,7 +164,6 @@ inputs: { "events": [], "want_more": false, - "cursor": {}, } ) ) @@ -242,6 +260,7 @@ inputs: state: api_token: ${SECRET_0} batch_size: 100 + initial_interval: 1h site_ids: "123" tags: - preserve_original_event diff --git a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.yml b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.yml index e31258064f6..377b063a59c 100644 --- a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.yml +++ b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.yml @@ -83,6 +83,7 @@ vars: -----END PRIVATE KEY----- data_stream: vars: + initial_interval: 1h interval: 30s batch_size: 100 max_executions: 1000 diff --git a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.expected b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.expected index 8ae6c0038b5..6dd2892f687 100644 --- a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.expected +++ b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.expected @@ -9,7 +9,7 @@ inputs: - config_version: 2 data_stream: dataset: sentinel_one.threat_event - interval: 24h + interval: 5m max_executions: 1000 program: |- ( @@ -22,8 +22,17 @@ inputs: state.url.trim_right("/") + "/web/api/v2.1/threats?" + { "skipCount": ["true"], "limit": [string(state.batch_size)], + "sortBy": ["updatedAt"], + "sortOrder": ["asc"], + "updatedAt__gte": [ + state.?cursor.?list_updated_at_gte.orValue( + state.?cursor.?last_timestamp.orValue( + (now - duration(state.initial_interval)).format(time_layout.RFC3339) + ) + ) + ], ?"siteIds": state.?site_ids.optMap(v, [string(v)]), - ?"cursor": state.?cursor.next_page.token.optMap(v, [v]), + ?"cursor": state.?cursor.?next_page.token.optMap(v, [v]), }.format_query() ).with( { @@ -40,6 +49,12 @@ inputs: ?"token": (body.?pagination.nextCursor.orValue(null) != null) ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": body.?pagination.nextCursor.orValue(null) != null, + "list_updated_at_gte": state.?cursor.?list_updated_at_gte.orValue( + state.?cursor.?last_timestamp.orValue( + (now - duration(state.initial_interval)).format(time_layout.RFC3339) + ) + ), + ?"last_timestamp": state.?cursor.last_timestamp, }, } ) @@ -58,7 +73,6 @@ inputs: }, }, "want_more": false, - "cursor": {}, } ) ) @@ -108,6 +122,11 @@ inputs: ?"token": has_more_events ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": state.?cursor.fetch_more.orValue(false), + "list_updated_at_gte": state.cursor.list_updated_at_gte, + ?"last_timestamp": has_more_events ? + state.?cursor.last_timestamp + : + state.cursor.worklist.data[0].?threatInfo.updatedAt.or(state.?cursor.last_timestamp), }, } ) @@ -134,7 +153,6 @@ inputs: { "events": [], "want_more": false, - "cursor": {}, } ) ) @@ -152,6 +170,7 @@ inputs: state: api_token: ${SECRET_0} batch_size: 1000 + initial_interval: 24h tags: - forwarded - sentinel_one-threat_event diff --git a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.yml b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.yml index ffd20d13b63..ab30cca6ec9 100644 --- a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.yml +++ b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.yml @@ -3,7 +3,8 @@ vars: api_token: test_api_token data_stream: vars: - interval: 24h + initial_interval: 24h + interval: 5m batch_size: 1000 max_executions: 1000 enable_request_tracer: false diff --git a/packages/sentinel_one/data_stream/threat_event/agent/stream/cel.yml.hbs b/packages/sentinel_one/data_stream/threat_event/agent/stream/cel.yml.hbs index 4ea4f9d3531..d3792581b6c 100644 --- a/packages/sentinel_one/data_stream/threat_event/agent/stream/cel.yml.hbs +++ b/packages/sentinel_one/data_stream/threat_event/agent/stream/cel.yml.hbs @@ -18,6 +18,7 @@ resource.url: {{url}} state: api_token: {{api_token}} batch_size: {{batch_size}} + initial_interval: {{initial_interval}} {{#if site_ids }} site_ids: !!str {{site_ids}} {{/if}} @@ -35,8 +36,17 @@ program: |- state.url.trim_right("/") + "/web/api/v2.1/threats?" + { "skipCount": ["true"], "limit": [string(state.batch_size)], + "sortBy": ["updatedAt"], + "sortOrder": ["asc"], + "updatedAt__gte": [ + state.?cursor.?list_updated_at_gte.orValue( + state.?cursor.?last_timestamp.orValue( + (now - duration(state.initial_interval)).format(time_layout.RFC3339) + ) + ) + ], ?"siteIds": state.?site_ids.optMap(v, [string(v)]), - ?"cursor": state.?cursor.next_page.token.optMap(v, [v]), + ?"cursor": state.?cursor.?next_page.token.optMap(v, [v]), }.format_query() ).with( { @@ -53,6 +63,12 @@ program: |- ?"token": (body.?pagination.nextCursor.orValue(null) != null) ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": body.?pagination.nextCursor.orValue(null) != null, + "list_updated_at_gte": state.?cursor.?list_updated_at_gte.orValue( + state.?cursor.?last_timestamp.orValue( + (now - duration(state.initial_interval)).format(time_layout.RFC3339) + ) + ), + ?"last_timestamp": state.?cursor.last_timestamp, }, } ) @@ -71,7 +87,6 @@ program: |- }, }, "want_more": false, - "cursor": {}, } ) ) @@ -121,6 +136,11 @@ program: |- ?"token": has_more_events ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": state.?cursor.fetch_more.orValue(false), + "list_updated_at_gte": state.cursor.list_updated_at_gte, + ?"last_timestamp": has_more_events ? + state.?cursor.last_timestamp + : + state.cursor.worklist.data[0].?threatInfo.updatedAt.or(state.?cursor.last_timestamp), }, } ) @@ -147,7 +167,6 @@ program: |- { "events": [], "want_more": false, - "cursor": {}, } ) ) diff --git a/packages/sentinel_one/data_stream/threat_event/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/threat_event/elasticsearch/ingest_pipeline/default.yml index 8b69f831eeb..e9df1d16e9e 100644 --- a/packages/sentinel_one/data_stream/threat_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one/data_stream/threat_event/elasticsearch/ingest_pipeline/default.yml @@ -45,6 +45,13 @@ processors: field: event.original tag: json_event_original target_field: json + - fingerprint: + tag: fingerprint_threat_event_dedup + fields: + - json.id + - json.createdAt + target_field: _id + ignore_missing: true - rename: field: json.id tag: rename_id diff --git a/packages/sentinel_one/data_stream/threat_event/manifest.yml b/packages/sentinel_one/data_stream/threat_event/manifest.yml index 742316a295b..cef9ebf5fc7 100644 --- a/packages/sentinel_one/data_stream/threat_event/manifest.yml +++ b/packages/sentinel_one/data_stream/threat_event/manifest.yml @@ -8,11 +8,19 @@ streams: enabled: false template_path: cel.yml.hbs vars: + - name: initial_interval + type: text + title: Initial Interval + description: How far back to look for threats on the first run. Supported units for this parameter are h/m/s. + default: 24h + multi: false + required: true + show_user: true - name: interval type: text title: Interval description: Duration between requests to the SentinelOne API. Supported units for this parameter are h/m/s. - default: 24h + default: 5m multi: false required: true show_user: true diff --git a/packages/sentinel_one/manifest.yml b/packages/sentinel_one/manifest.yml index 088ed8b31c9..7f8b4a5d61a 100644 --- a/packages/sentinel_one/manifest.yml +++ b/packages/sentinel_one/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: sentinel_one title: SentinelOne -version: "2.11.0" +version: "2.11.1" description: Collect logs from SentinelOne with Elastic Agent. type: integration categories: