Skip to content

sentinel_one: fix duplicate event ingestion in threat_event and application_risk #20211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
efd6 merged 1 commit into from
Jul 20, 2026
Merged

sentinel_one: fix duplicate event ingestion in threat_event and application_risk #20211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions packages/sentinel_one/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
# newer versions go on top
- version: "2.11.1"
changes:
- description: Add time watermark and fingerprint deduplication to the threat event data stream to prevent duplicate ingestion.
type: bugfix
link: https://github.com/elastic/integrations/pull/20211
- description: Add fingerprint deduplication to the application risk data stream to prevent duplicate ingestion.
type: bugfix
link: https://github.com/elastic/integrations/pull/20211
- version: "2.11.0"
changes:
- description: |
Expand Down
7 changes: 7 additions & 0 deletions packages/sentinel_one/data_stream/application_risk/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ processors:
field: event.original
tag: json_event_original
target_field: json
- fingerprint:
tag: fingerprint_application_risk_dedup
fields:
- json.id
Comment thread
mohitjha-elastic marked this conversation as resolved.
- json.status
target_field: _id
ignore_missing: true
- set:
field: event.kind
tag: set_event_kind
Expand Down
25 changes: 22 additions & 3 deletions packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,17 @@ inputs:
state.url.trim_right("/") + "/web/api/v2.1/threats?" + {
"skipCount": ["true"],
"limit": [string(state.batch_size)],
"sortBy": ["updatedAt"],
"sortOrder": ["asc"],
"updatedAt__gte": [
state.?cursor.?list_updated_at_gte.orValue(
state.?cursor.?last_timestamp.orValue(
(now - duration(state.initial_interval)).format(time_layout.RFC3339)
)
)
],
?"siteIds": state.?site_ids.optMap(v, [string(v)]),
?"cursor": state.?cursor.next_page.token.optMap(v, [v]),
?"cursor": state.?cursor.?next_page.token.optMap(v, [v]),
}.format_query()
).with(
{
Expand All @@ -51,6 +60,12 @@ inputs:
?"token": (body.?pagination.nextCursor.orValue(null) != null) ? optional.of(body.pagination.nextCursor) : optional.none(),
},
"fetch_more": body.?pagination.nextCursor.orValue(null) != null,
"list_updated_at_gte": state.?cursor.?list_updated_at_gte.orValue(
state.?cursor.?last_timestamp.orValue(
(now - duration(state.initial_interval)).format(time_layout.RFC3339)
)
),
?"last_timestamp": state.?cursor.last_timestamp,
},
}
)
Expand All @@ -69,7 +84,6 @@ inputs:
},
},
"want_more": false,
"cursor": {},
}
)
)
Expand Down Expand Up @@ -119,6 +133,11 @@ inputs:
?"token": has_more_events ? optional.of(body.pagination.nextCursor) : optional.none(),
},
"fetch_more": state.?cursor.fetch_more.orValue(false),
"list_updated_at_gte": state.cursor.list_updated_at_gte,
?"last_timestamp": has_more_events ?
state.?cursor.last_timestamp
:
state.cursor.worklist.data[0].?threatInfo.updatedAt.or(state.?cursor.last_timestamp),
},
}
)
Expand All @@ -145,7 +164,6 @@ inputs:
{
"events": [],
"want_more": false,
"cursor": {},
}
)
)
Expand Down Expand Up @@ -242,6 +260,7 @@ inputs:
state:
api_token: ${SECRET_0}
batch_size: 100
initial_interval: 1h
site_ids: "123"
tags:
- preserve_original_event
Expand Down
1 change: 1 addition & 0 deletions packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ vars:
-----END PRIVATE KEY-----
data_stream:
vars:
initial_interval: 1h
interval: 30s
batch_size: 100
max_executions: 1000
Expand Down
27 changes: 23 additions & 4 deletions packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ inputs:
- config_version: 2
data_stream:
dataset: sentinel_one.threat_event
interval: 24h
interval: 5m
max_executions: 1000
program: |-
(
Expand All @@ -22,8 +22,17 @@ inputs:
state.url.trim_right("/") + "/web/api/v2.1/threats?" + {
"skipCount": ["true"],
"limit": [string(state.batch_size)],
"sortBy": ["updatedAt"],
"sortOrder": ["asc"],
"updatedAt__gte": [
state.?cursor.?list_updated_at_gte.orValue(
state.?cursor.?last_timestamp.orValue(
(now - duration(state.initial_interval)).format(time_layout.RFC3339)
)
)
],
?"siteIds": state.?site_ids.optMap(v, [string(v)]),
?"cursor": state.?cursor.next_page.token.optMap(v, [v]),
?"cursor": state.?cursor.?next_page.token.optMap(v, [v]),
}.format_query()
).with(
{
Expand All @@ -40,6 +49,12 @@ inputs:
?"token": (body.?pagination.nextCursor.orValue(null) != null) ? optional.of(body.pagination.nextCursor) : optional.none(),
},
"fetch_more": body.?pagination.nextCursor.orValue(null) != null,
"list_updated_at_gte": state.?cursor.?list_updated_at_gte.orValue(
state.?cursor.?last_timestamp.orValue(
(now - duration(state.initial_interval)).format(time_layout.RFC3339)
)
),
?"last_timestamp": state.?cursor.last_timestamp,
},
}
)
Expand All @@ -58,7 +73,6 @@ inputs:
},
},
"want_more": false,
"cursor": {},
}
)
)
Expand Down Expand Up @@ -108,6 +122,11 @@ inputs:
?"token": has_more_events ? optional.of(body.pagination.nextCursor) : optional.none(),
},
"fetch_more": state.?cursor.fetch_more.orValue(false),
"list_updated_at_gte": state.cursor.list_updated_at_gte,
?"last_timestamp": has_more_events ?
state.?cursor.last_timestamp
:
state.cursor.worklist.data[0].?threatInfo.updatedAt.or(state.?cursor.last_timestamp),
},
}
)
Expand All @@ -134,7 +153,6 @@ inputs:
{
"events": [],
"want_more": false,
"cursor": {},
}
)
)
Expand All @@ -152,6 +170,7 @@ inputs:
state:
api_token: ${SECRET_0}
batch_size: 1000
initial_interval: 24h
tags:
- forwarded
- sentinel_one-threat_event
Expand Down
3 changes: 2 additions & 1 deletion packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ vars:
api_token: test_api_token
data_stream:
vars:
interval: 24h
initial_interval: 24h
interval: 5m
batch_size: 1000
max_executions: 1000
enable_request_tracer: false
Expand Down
25 changes: 22 additions & 3 deletions packages/sentinel_one/data_stream/threat_event/agent/stream/cel.yml.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ resource.url: {{url}}
state:
api_token: {{api_token}}
batch_size: {{batch_size}}
initial_interval: {{initial_interval}}
{{#if site_ids }}
site_ids: !!str {{site_ids}}
{{/if}}
Expand All @@ -35,8 +36,17 @@ program: |-
state.url.trim_right("/") + "/web/api/v2.1/threats?" + {
"skipCount": ["true"],
"limit": [string(state.batch_size)],
"sortBy": ["updatedAt"],
"sortOrder": ["asc"],
"updatedAt__gte": [
state.?cursor.?list_updated_at_gte.orValue(
state.?cursor.?last_timestamp.orValue(
(now - duration(state.initial_interval)).format(time_layout.RFC3339)
)
)
],
?"siteIds": state.?site_ids.optMap(v, [string(v)]),
?"cursor": state.?cursor.next_page.token.optMap(v, [v]),
?"cursor": state.?cursor.?next_page.token.optMap(v, [v]),
}.format_query()
).with(
{
Expand All @@ -53,6 +63,12 @@ program: |-
?"token": (body.?pagination.nextCursor.orValue(null) != null) ? optional.of(body.pagination.nextCursor) : optional.none(),
},
"fetch_more": body.?pagination.nextCursor.orValue(null) != null,
"list_updated_at_gte": state.?cursor.?list_updated_at_gte.orValue(
state.?cursor.?last_timestamp.orValue(
(now - duration(state.initial_interval)).format(time_layout.RFC3339)
)
),
?"last_timestamp": state.?cursor.last_timestamp,
},
}
)
Expand All @@ -71,7 +87,6 @@ program: |-
},
},
"want_more": false,
"cursor": {},
}
)
)
Expand Down Expand Up @@ -121,6 +136,11 @@ program: |-
?"token": has_more_events ? optional.of(body.pagination.nextCursor) : optional.none(),
},
"fetch_more": state.?cursor.fetch_more.orValue(false),
"list_updated_at_gte": state.cursor.list_updated_at_gte,
?"last_timestamp": has_more_events ?
state.?cursor.last_timestamp
:
state.cursor.worklist.data[0].?threatInfo.updatedAt.or(state.?cursor.last_timestamp),
},
}
)
Expand All @@ -147,7 +167,6 @@ program: |-
{
"events": [],
"want_more": false,
"cursor": {},
}
)
)
Expand Down
7 changes: 7 additions & 0 deletions packages/sentinel_one/data_stream/threat_event/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ processors:
field: event.original
tag: json_event_original
target_field: json
- fingerprint:
tag: fingerprint_threat_event_dedup
fields:
- json.id
- json.createdAt
target_field: _id
ignore_missing: true
- rename:
field: json.id
tag: rename_id
Expand Down
10 changes: 9 additions & 1 deletion packages/sentinel_one/data_stream/threat_event/manifest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,19 @@ streams:
enabled: false
template_path: cel.yml.hbs
vars:
- name: initial_interval
type: text
title: Initial Interval
description: How far back to look for threats on the first run. Supported units for this parameter are h/m/s.
default: 24h
multi: false
required: true
show_user: true
- name: interval
type: text
title: Interval
description: Duration between requests to the SentinelOne API. Supported units for this parameter are h/m/s.
default: 24h
default: 5m
multi: false
required: true
show_user: true
Expand Down
2 changes: 1 addition & 1 deletion packages/sentinel_one/manifest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
format_version: "3.4.0"
name: sentinel_one
title: SentinelOne
version: "2.11.0"
version: "2.11.1"
description: Collect logs from SentinelOne with Elastic Agent.
type: integration
categories:
Expand Down
Loading