From 67187963a9334f5d01eaf0309e3750cf46dea03c Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Mon, 20 Jul 2026 13:06:27 +0930 Subject: [PATCH] entityanalytics_{ad,entra_id,okta}: add agentless deployment mode and minimal-state sync Enable agentless deployment for all three entity analytics packages by bumping format_version to 3.3.2, adding deployment_modes to the policy templates, and introducing a use_minimal_state configuration variable (default true, hidden from users in both deployment modes). Existing system tests are pinned to use_minimal_state: false to keep them on the legacy sync path. New script tests verify the minimal-state path routes entities to per-type data streams (AD and EntraID). Okta minimal-state script tests are blocked on elastic/beats#52061 (the entcollect Okta provider hardcodes https:// but does not accept custom TLS config, preventing testing against a self-signed mock). The EntraID mock gains /v1.0/groups and /v1.0/groups/{id}/members endpoints required by the entcollect provider's non-delta group fetch. --- packages/entityanalytics_ad/changelog.yml | 5 ++ .../_dev/test/scripts/minimal_state.txt | 69 +++++++++++++++++++ .../test/scripts/minimal_state_default.txt | 67 ++++++++++++++++++ .../_dev/test/scripts/routed_data_streams.txt | 7 +- .../_dev/test/system/test-default-config.yml | 1 + .../test/system/test-special-chars-config.yml | 1 + .../agent/stream/entity-analytics.yml.hbs | 3 + .../data_stream/entity/manifest.yml | 13 ++++ packages/entityanalytics_ad/manifest.yml | 15 +++- .../_dev/deploy/docker/files/config.yml | 18 +++++ .../entityanalytics_entra_id/changelog.yml | 5 ++ .../_dev/test/policy/test-default.expected | 1 + .../test-intune-with-custom-options.expected | 1 + .../_dev/test/policy/test-intune.expected | 1 + .../entity/_dev/test/scripts/env.txt | 10 +++ .../_dev/test/scripts/minimal_state.txt | 57 +++++++++++++++ .../test/scripts/minimal_state_default.txt | 55 +++++++++++++++ .../test-app-role-assignments-config.yml | 1 + .../_dev/test/system/test-default-config.yml | 1 + .../system/test-direct-reports-config.yml | 1 + .../_dev/test/system/test-mfa-config.yml | 1 + .../system/test-sign-in-activity-config.yml | 1 + .../agent/stream/entity-analytics.yml.hbs | 3 + .../data_stream/entity/manifest.yml | 15 ++++ .../entityanalytics_entra_id/manifest.yml | 15 +++- packages/entityanalytics_okta/changelog.yml | 5 ++ .../entity/_dev/test/scripts/env.txt | 10 +++ .../_dev/test/system/test-default-config.yml | 1 + .../system/test-enrich-devices-config.yml | 1 + .../system/test-enrich-factors-config.yml | 1 + .../test/system/test-enrich-perms-config.yml | 1 + .../test/system/test-enrich-roles-config.yml | 1 + .../system/test-enrich-supervises-config.yml | 1 + .../test/system/test-oauth2-oin-config.yml | 1 + .../agent/stream/entity-analytics.yml.hbs | 3 + .../data_stream/entity/manifest.yml | 17 +++++ packages/entityanalytics_okta/manifest.yml | 15 +++- 37 files changed, 410 insertions(+), 14 deletions(-) create mode 100644 packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/minimal_state.txt create mode 100644 packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/minimal_state_default.txt create mode 100644 packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/env.txt create mode 100644 packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/minimal_state.txt create mode 100644 packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/minimal_state_default.txt create mode 100644 packages/entityanalytics_okta/data_stream/entity/_dev/test/scripts/env.txt diff --git a/packages/entityanalytics_ad/changelog.yml b/packages/entityanalytics_ad/changelog.yml index 604e7824904..a9c08ba3695 100644 --- a/packages/entityanalytics_ad/changelog.yml +++ b/packages/entityanalytics_ad/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.22.0" + changes: + - description: Add agentless deployment mode and minimal-state sync support. + type: enhancement + link: https://github.com/elastic/integrations/pull/20212 - version: "0.21.0" changes: - description: Add the Advanced Analytics (UEBA) subcategory diff --git a/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/minimal_state.txt b/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/minimal_state.txt new file mode 100644 index 00000000000..bdcd999c8f5 --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/minimal_state.txt @@ -0,0 +1,69 @@ +# Verify the minimal-state sync path routes entity documents to per-type data +# streams without producing marker events in the entity data stream. +# +# This test explicitly sets use_minimal_state: true. +# +# The mock LDAP server provides: +# - 1 user (Administrator) → entityanalytics_ad.user +# - 1 device (WORKSTATION-01) → entityanalytics_ad.device +# - 2 empty groups (Decommissioned, Contractors) → entityanalytics_ad.group + +[!external_stack] skip 'Skipping external stack test.' +[!exec:jq] skip 'Skipping test requiring absent jq command' + +use_stack -profile ${CONFIG_PROFILES}/${PROFILE} + +install_agent -profile ${CONFIG_PROFILES}/${PROFILE} -network_name NETWORK_NAME + +exec ln -s ${PACKAGE_ROOT}/_dev/deploy/docker ldap-mock-service +docker_up -profile ${CONFIG_PROFILES}/${PROFILE} -network ${NETWORK_NAME} ldap-mock-service + +add_package -profile ${CONFIG_PROFILES}/${PROFILE} + +add_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} test_config.yaml ENTITY_DS + +# Verify user documents arrive in the user data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 1 -timeout 5m logs-entityanalytics_ad.user-* +cp stdout user_docs.json +exec jq '[.hits.hits[]._source.user.id] | unique[]' user_docs.json +stdout 'S-1-5-21-374105552-1189110957-4047727897-500' +exec jq '[.hits.hits[]._source.user.name] | unique[]' user_docs.json +stdout 'Administrator' + +# Verify device documents arrive in the device data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 1 -timeout 2m logs-entityanalytics_ad.device-* +cp stdout device_docs.json +exec jq '[.hits.hits[]._source.device.id] | unique[]' device_docs.json +stdout 'S-1-5-21-374105552-1189110957-4047727897-1001' + +# Verify empty group documents arrive in the group data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 2 -timeout 2m logs-entityanalytics_ad.group-* +cp stdout group_docs.json +exec jq '[.hits.hits[]._source.group.id] | unique | sort[]' group_docs.json +stdout 'S-1-5-21-374105552-1189110957-4047727897-1101' +stdout 'S-1-5-21-374105552-1189110957-4047727897-1102' +exec jq '[.hits.hits[]._source.group.name] | unique | sort[]' group_docs.json +stdout 'Contractors' +stdout 'Decommissioned' + +# The entity data stream should have no documents (no markers in minimal mode). +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 0 -timeout 30s ${ENTITY_DS} + +# Clean up. +remove_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} ${ENTITY_DS} +uninstall_agent -profile ${CONFIG_PROFILES}/${PROFILE} -timeout 1m +docker_down ldap-mock-service + +-- test_config.yaml -- +input: entity-analytics +data_stream: + vars: + ad_base_dn: "DC=testserver,DC=local" + ad_url: "ldap://svc-ldap-mock-service:10389" + ad_user: "cn=admin,dc=testserver,dc=local" + ad_password: "password" + dataset: all + use_minimal_state: true + include_empty_groups: true + sync_interval: 24h + update_interval: 15m diff --git a/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/minimal_state_default.txt b/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/minimal_state_default.txt new file mode 100644 index 00000000000..1a66ccfba6f --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/minimal_state_default.txt @@ -0,0 +1,67 @@ +# Verify that the default configuration (use_minimal_state not set) uses the +# minimal-state sync path. The package default is true, so the behavior should +# be identical to setting it explicitly. +# +# The mock LDAP server provides: +# - 1 user (Administrator) → entityanalytics_ad.user +# - 1 device (WORKSTATION-01) → entityanalytics_ad.device +# - 2 empty groups (Decommissioned, Contractors) → entityanalytics_ad.group + +[!external_stack] skip 'Skipping external stack test.' +[!exec:jq] skip 'Skipping test requiring absent jq command' + +use_stack -profile ${CONFIG_PROFILES}/${PROFILE} + +install_agent -profile ${CONFIG_PROFILES}/${PROFILE} -network_name NETWORK_NAME + +exec ln -s ${PACKAGE_ROOT}/_dev/deploy/docker ldap-mock-service +docker_up -profile ${CONFIG_PROFILES}/${PROFILE} -network ${NETWORK_NAME} ldap-mock-service + +add_package -profile ${CONFIG_PROFILES}/${PROFILE} + +add_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} test_config.yaml ENTITY_DS + +# Verify user documents arrive in the user data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 1 -timeout 5m logs-entityanalytics_ad.user-* +cp stdout user_docs.json +exec jq '[.hits.hits[]._source.user.id] | unique[]' user_docs.json +stdout 'S-1-5-21-374105552-1189110957-4047727897-500' +exec jq '[.hits.hits[]._source.user.name] | unique[]' user_docs.json +stdout 'Administrator' + +# Verify device documents arrive in the device data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 1 -timeout 2m logs-entityanalytics_ad.device-* +cp stdout device_docs.json +exec jq '[.hits.hits[]._source.device.id] | unique[]' device_docs.json +stdout 'S-1-5-21-374105552-1189110957-4047727897-1001' + +# Verify empty group documents arrive in the group data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 2 -timeout 2m logs-entityanalytics_ad.group-* +cp stdout group_docs.json +exec jq '[.hits.hits[]._source.group.id] | unique | sort[]' group_docs.json +stdout 'S-1-5-21-374105552-1189110957-4047727897-1101' +stdout 'S-1-5-21-374105552-1189110957-4047727897-1102' +exec jq '[.hits.hits[]._source.group.name] | unique | sort[]' group_docs.json +stdout 'Contractors' +stdout 'Decommissioned' + +# The entity data stream should have no documents (no markers in minimal mode). +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 0 -timeout 30s ${ENTITY_DS} + +# Clean up. +remove_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} ${ENTITY_DS} +uninstall_agent -profile ${CONFIG_PROFILES}/${PROFILE} -timeout 1m +docker_down ldap-mock-service + +-- test_config.yaml -- +input: entity-analytics +data_stream: + vars: + ad_base_dn: "DC=testserver,DC=local" + ad_url: "ldap://svc-ldap-mock-service:10389" + ad_user: "cn=admin,dc=testserver,dc=local" + ad_password: "password" + dataset: all + include_empty_groups: true + sync_interval: 24h + update_interval: 15m diff --git a/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/routed_data_streams.txt b/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/routed_data_streams.txt index 09a0093152d..50f4aa57055 100644 --- a/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/routed_data_streams.txt +++ b/packages/entityanalytics_ad/data_stream/entity/_dev/test/scripts/routed_data_streams.txt @@ -41,14 +41,10 @@ exec jq '[.hits.hits[]._source.user.name] | unique[]' user_docs.json stdout 'Administrator' # Verify device documents were routed to the device data stream. -# NOTE: The device pipeline (device.yml) has a pre-existing bug — it references -# activedirectory.device.* but the beats input puts device attributes under -# activedirectory.user.*. The pipeline fails, so device.id retains the DN set -# by the input rather than being overwritten with the SID. get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 1 -timeout 2m logs-entityanalytics_ad.device-* cp stdout device_docs.json exec jq '[.hits.hits[]._source.device.id] | unique[]' device_docs.json -stdout 'WORKSTATION-01' +stdout 'S-1-5-21-374105552-1189110957-4047727897-1001' # Verify empty group documents were routed to the group data stream. get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 2 -timeout 2m logs-entityanalytics_ad.group-* @@ -74,6 +70,7 @@ data_stream: ad_user: "cn=admin,dc=testserver,dc=local" ad_password: "password" dataset: all + use_minimal_state: false include_empty_groups: true sync_interval: 24h update_interval: 15m diff --git a/packages/entityanalytics_ad/data_stream/entity/_dev/test/system/test-default-config.yml b/packages/entityanalytics_ad/data_stream/entity/_dev/test/system/test-default-config.yml index 0f6dc13bc20..fd086599858 100644 --- a/packages/entityanalytics_ad/data_stream/entity/_dev/test/system/test-default-config.yml +++ b/packages/entityanalytics_ad/data_stream/entity/_dev/test/system/test-default-config.yml @@ -11,5 +11,6 @@ data_stream: include_empty_groups: true sync_interval: 24h update_interval: 15m + use_minimal_state: false assert: hit_count: 2 diff --git a/packages/entityanalytics_ad/data_stream/entity/_dev/test/system/test-special-chars-config.yml b/packages/entityanalytics_ad/data_stream/entity/_dev/test/system/test-special-chars-config.yml index 6e57de165fe..4a927da0146 100644 --- a/packages/entityanalytics_ad/data_stream/entity/_dev/test/system/test-special-chars-config.yml +++ b/packages/entityanalytics_ad/data_stream/entity/_dev/test/system/test-special-chars-config.yml @@ -11,5 +11,6 @@ data_stream: include_empty_groups: true sync_interval: 24h update_interval: 15m + use_minimal_state: false assert: hit_count: 2 diff --git a/packages/entityanalytics_ad/data_stream/entity/agent/stream/entity-analytics.yml.hbs b/packages/entityanalytics_ad/data_stream/entity/agent/stream/entity-analytics.yml.hbs index 5d73fe3dc8d..6b4c1324027 100644 --- a/packages/entityanalytics_ad/data_stream/entity/agent/stream/entity-analytics.yml.hbs +++ b/packages/entityanalytics_ad/data_stream/entity/agent/stream/entity-analytics.yml.hbs @@ -1,4 +1,7 @@ provider: activedirectory +{{#if use_minimal_state}} +use_minimal_state: true +{{/if}} sync_interval: {{sync_interval}} update_interval: {{update_interval}} ad_base_dn: {{ad_base_dn}} diff --git a/packages/entityanalytics_ad/data_stream/entity/manifest.yml b/packages/entityanalytics_ad/data_stream/entity/manifest.yml index e86cb15631f..43752067378 100644 --- a/packages/entityanalytics_ad/data_stream/entity/manifest.yml +++ b/packages/entityanalytics_ad/data_stream/entity/manifest.yml @@ -174,6 +174,19 @@ streams: show_user: false description: >- Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: use_minimal_state + type: bool + title: Use minimal state + description: >- + Use the minimal-state sync implementation. Required for agentless + deployments. Uses reduced local storage and supports ES-backed state. + multi: false + required: false + show_user: false + default: true + hide_in_deployment_modes: + - default + - agentless elasticsearch: dynamic_dataset: true dynamic_namespace: true diff --git a/packages/entityanalytics_ad/manifest.yml b/packages/entityanalytics_ad/manifest.yml index f171532289a..985a402494c 100644 --- a/packages/entityanalytics_ad/manifest.yml +++ b/packages/entityanalytics_ad/manifest.yml @@ -1,7 +1,7 @@ -format_version: "3.0.2" +format_version: "3.3.2" name: entityanalytics_ad title: Active Directory Entity Analytics -version: "0.21.0" +version: "0.22.0" description: "Collect User Identities from Active Directory Entity with Elastic Agent." type: integration categories: @@ -9,7 +9,7 @@ categories: - advanced_analytics_ueba conditions: kibana: - version: "^9.4.0" + version: "^9.5.0" elastic: subscription: "basic" screenshots: [] @@ -22,6 +22,15 @@ policy_templates: - name: entity title: Identities description: Collect identities. + deployment_modes: + default: + enabled: true + agentless: + enabled: true + release: beta + organization: security + division: engineering + team: security-service-integrations inputs: - type: entity-analytics title: Collect identities diff --git a/packages/entityanalytics_entra_id/_dev/deploy/docker/files/config.yml b/packages/entityanalytics_entra_id/_dev/deploy/docker/files/config.yml index 62aee0df355..76ee684af17 100644 --- a/packages/entityanalytics_entra_id/_dev/deploy/docker/files/config.yml +++ b/packages/entityanalytics_entra_id/_dev/deploy/docker/files/config.yml @@ -17,6 +17,24 @@ rules: - status_code: 200 body: |- {"@odata.deltaLink":"http://localhost:8080/v1.0/groups/delta?$skiptoken=1","value":[{"id":"e7089e3a-2c83-4f08-8280-7530ed39b6ca","name":"Group 5202","members@delta":[{"id":"aa534e49-edfd-4541-8256-8bbf34f122b4","type":"#microsoft.graph.user"},{"id":"feb6a386-612a-4ed1-9b13-2adc73074a19","type":"#microsoft.graph.user"}]},{"id":"526588ce-2828-4cb1-9c9b-e57026e94b82","name":"Group 16739","members@delta":[{"id":"aa534e49-edfd-4541-8256-8bbf34f122b4","type":"#microsoft.graph.user"}]}]} + - path: /v1.0/groups + methods: ["GET"] + responses: + - status_code: 200 + body: |- + {"value":[{"id":"e7089e3a-2c83-4f08-8280-7530ed39b6ca","displayName":"Group 5202"},{"id":"526588ce-2828-4cb1-9c9b-e57026e94b82","displayName":"Group 16739"}]} + - path: /v1.0/groups/e7089e3a-2c83-4f08-8280-7530ed39b6ca/members + methods: ["GET"] + responses: + - status_code: 200 + body: |- + {"value":[{"id":"aa534e49-edfd-4541-8256-8bbf34f122b4","@odata.type":"#microsoft.graph.user"},{"id":"feb6a386-612a-4ed1-9b13-2adc73074a19","@odata.type":"#microsoft.graph.user"}]} + - path: /v1.0/groups/526588ce-2828-4cb1-9c9b-e57026e94b82/members + methods: ["GET"] + responses: + - status_code: 200 + body: |- + {"value":[{"id":"aa534e49-edfd-4541-8256-8bbf34f122b4","@odata.type":"#microsoft.graph.user"}]} - path: /v1.0/reports/authenticationMethods/userRegistrationDetails methods: ["GET"] responses: diff --git a/packages/entityanalytics_entra_id/changelog.yml b/packages/entityanalytics_entra_id/changelog.yml index 421c2e6d8dd..5acecedc9a5 100644 --- a/packages/entityanalytics_entra_id/changelog.yml +++ b/packages/entityanalytics_entra_id/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.12.0" + changes: + - description: Add agentless deployment mode and minimal-state sync support. + type: enhancement + link: https://github.com/elastic/integrations/pull/20212 - version: "1.11.0" changes: - description: Add the Advanced Analytics (UEBA) subcategory diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-default.expected b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-default.expected index fdb4891d60d..4349728cfda 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-default.expected +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-default.expected @@ -26,6 +26,7 @@ inputs: tenant_id: test-tenant-id timeout: 30s update_interval: 15m + use_minimal_state: true type: entity-analytics use_output: default output_permissions: diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-intune-with-custom-options.expected b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-intune-with-custom-options.expected index 2bce47fada7..0928aa0668c 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-intune-with-custom-options.expected +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-intune-with-custom-options.expected @@ -52,6 +52,7 @@ inputs: tenant_id: test-tenant-id timeout: 30s update_interval: 15m + use_minimal_state: true type: entity-analytics use_output: default output_permissions: diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-intune.expected b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-intune.expected index be94442b9f7..4a744ea89a7 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-intune.expected +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/policy/test-intune.expected @@ -47,6 +47,7 @@ inputs: tenant_id: test-tenant-id timeout: 30s update_interval: 15m + use_minimal_state: true type: entity-analytics use_output: default output_permissions: diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/env.txt b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/env.txt new file mode 100644 index 00000000000..0068e71def6 --- /dev/null +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/env.txt @@ -0,0 +1,10 @@ +[!exec:echo] skip 'Skipping test requiring absent echo command' + +exec echo ${PACKAGE_NAME} +stdout '^entityanalytics_entra_id$' + +exec echo ${DATA_STREAM} +stdout '^entity$' + +exec echo ${CURRENT_VERSION} +stdout '^[0-9]+\.[0-9]+\.[0-9]+$' diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/minimal_state.txt b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/minimal_state.txt new file mode 100644 index 00000000000..f13d4397931 --- /dev/null +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/minimal_state.txt @@ -0,0 +1,57 @@ +# Verify the minimal-state sync path routes Entra ID entities to per-type data +# streams without producing marker events in the entity data stream. +# +# This test explicitly sets use_minimal_state: true. +# +# The mock Graph API provides: +# - 2 users → entityanalytics_entra_id.user +# - 1 device (object id c9d9f9b3-…) → entityanalytics_entra_id.device + +[!external_stack] skip 'Skipping external stack test.' +[!exec:jq] skip 'Skipping test requiring absent jq command' + +use_stack -profile ${CONFIG_PROFILES}/${PROFILE} + +install_agent -profile ${CONFIG_PROFILES}/${PROFILE} -network_name NETWORK_NAME + +exec ln -s ${PACKAGE_ROOT}/_dev/deploy/docker entra_id +docker_up -profile ${CONFIG_PROFILES}/${PROFILE} -network ${NETWORK_NAME} entra_id + +add_package -profile ${CONFIG_PROFILES}/${PROFILE} + +add_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} test_config.yaml ENTITY_DS + +# Verify user documents arrive in the user data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 2 -timeout 5m logs-entityanalytics_entra_id.user-* +cp stdout user_docs.json +exec jq '[.hits.hits[]._source.user.id] | sort[]' user_docs.json +stdout 'aa534e49-edfd-4541-8256-8bbf34f122b4' +stdout 'feb6a386-612a-4ed1-9b13-2adc73074a19' + +# Verify device documents arrive in the device data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 1 -timeout 2m logs-entityanalytics_entra_id.device-* +cp stdout device_docs.json +exec jq '[.hits.hits[]._source.device.id] | unique[]' device_docs.json +stdout 'c9d9f9b3-0c91-4080-b392-78f775903b3a' + +# The entity data stream should have no documents (no markers in minimal mode). +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 0 -timeout 30s ${ENTITY_DS} + +# Clean up. +remove_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} ${ENTITY_DS} +uninstall_agent -profile ${CONFIG_PROFILES}/${PROFILE} -timeout 1m +docker_down entra_id + +-- test_config.yaml -- +input: entity-analytics +data_stream: + vars: + tenant_id: TENANT_ID + client_id: CLIENT_ID + secret: foobar + login_endpoint: http://svc-entra_id:8080 + api_endpoint: http://svc-entra_id:8080/v1.0 + dataset: all + use_minimal_state: true + sync_interval: 24h + update_interval: 15m diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/minimal_state_default.txt b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/minimal_state_default.txt new file mode 100644 index 00000000000..98ed15a0272 --- /dev/null +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/scripts/minimal_state_default.txt @@ -0,0 +1,55 @@ +# Verify that the default configuration (use_minimal_state not set) uses the +# minimal-state sync path. The package default is true, so the behavior should +# be identical to setting it explicitly. +# +# The mock Graph API provides: +# - 2 users → entityanalytics_entra_id.user +# - 1 device (object id c9d9f9b3-…) → entityanalytics_entra_id.device + +[!external_stack] skip 'Skipping external stack test.' +[!exec:jq] skip 'Skipping test requiring absent jq command' + +use_stack -profile ${CONFIG_PROFILES}/${PROFILE} + +install_agent -profile ${CONFIG_PROFILES}/${PROFILE} -network_name NETWORK_NAME + +exec ln -s ${PACKAGE_ROOT}/_dev/deploy/docker entra_id +docker_up -profile ${CONFIG_PROFILES}/${PROFILE} -network ${NETWORK_NAME} entra_id + +add_package -profile ${CONFIG_PROFILES}/${PROFILE} + +add_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} test_config.yaml ENTITY_DS + +# Verify user documents arrive in the user data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 2 -timeout 5m logs-entityanalytics_entra_id.user-* +cp stdout user_docs.json +exec jq '[.hits.hits[]._source.user.id] | sort[]' user_docs.json +stdout 'aa534e49-edfd-4541-8256-8bbf34f122b4' +stdout 'feb6a386-612a-4ed1-9b13-2adc73074a19' + +# Verify device documents arrive in the device data stream. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 1 -timeout 2m logs-entityanalytics_entra_id.device-* +cp stdout device_docs.json +exec jq '[.hits.hits[]._source.device.id] | unique[]' device_docs.json +stdout 'c9d9f9b3-0c91-4080-b392-78f775903b3a' + +# The entity data stream should have no documents (no markers in minimal mode). +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 0 -timeout 30s ${ENTITY_DS} + +# Clean up. +remove_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} ${ENTITY_DS} +uninstall_agent -profile ${CONFIG_PROFILES}/${PROFILE} -timeout 1m +docker_down entra_id + +-- test_config.yaml -- +input: entity-analytics +data_stream: + vars: + tenant_id: TENANT_ID + client_id: CLIENT_ID + secret: foobar + login_endpoint: http://svc-entra_id:8080 + api_endpoint: http://svc-entra_id:8080/v1.0 + dataset: all + sync_interval: 24h + update_interval: 15m diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-app-role-assignments-config.yml b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-app-role-assignments-config.yml index 7d4f603f9cb..a1fff637fcc 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-app-role-assignments-config.yml +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-app-role-assignments-config.yml @@ -11,3 +11,4 @@ data_stream: user_app_role_assignments: true enable_request_tracer: true preserve_original_event: true + use_minimal_state: false diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-default-config.yml b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-default-config.yml index 04fa5019d57..d08ca45817c 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-default-config.yml +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-default-config.yml @@ -10,3 +10,4 @@ data_stream: dataset: all enable_request_tracer: true preserve_original_event: true + use_minimal_state: false diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-direct-reports-config.yml b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-direct-reports-config.yml index 7b4b0ddc8a0..7224216f3c2 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-direct-reports-config.yml +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-direct-reports-config.yml @@ -11,3 +11,4 @@ data_stream: user_direct_reports: true enable_request_tracer: true preserve_original_event: true + use_minimal_state: false diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-mfa-config.yml b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-mfa-config.yml index 99c3c8d574f..87af02d9d65 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-mfa-config.yml +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-mfa-config.yml @@ -11,3 +11,4 @@ data_stream: enrich_with_mfa: true enable_request_tracer: true preserve_original_event: true + use_minimal_state: false diff --git a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-sign-in-activity-config.yml b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-sign-in-activity-config.yml index 21b98b9f7dc..c8fbaf1de96 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-sign-in-activity-config.yml +++ b/packages/entityanalytics_entra_id/data_stream/entity/_dev/test/system/test-sign-in-activity-config.yml @@ -11,3 +11,4 @@ data_stream: user_sign_in_activity: true enable_request_tracer: true preserve_original_event: true + use_minimal_state: false diff --git a/packages/entityanalytics_entra_id/data_stream/entity/agent/stream/entity-analytics.yml.hbs b/packages/entityanalytics_entra_id/data_stream/entity/agent/stream/entity-analytics.yml.hbs index 6d40d705206..46bb4735586 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/agent/stream/entity-analytics.yml.hbs +++ b/packages/entityanalytics_entra_id/data_stream/entity/agent/stream/entity-analytics.yml.hbs @@ -1,4 +1,7 @@ provider: azure-ad +{{#if use_minimal_state}} +use_minimal_state: true +{{/if}} {{#if enable_request_tracer}} tracer.filename: "../../logs/azure-ad/http-request-trace-*.ndjson" tracer.maxbackups: 5 diff --git a/packages/entityanalytics_entra_id/data_stream/entity/manifest.yml b/packages/entityanalytics_entra_id/data_stream/entity/manifest.yml index 5bb81d2e1de..62ce6f9f08d 100644 --- a/packages/entityanalytics_entra_id/data_stream/entity/manifest.yml +++ b/packages/entityanalytics_entra_id/data_stream/entity/manifest.yml @@ -187,6 +187,8 @@ streams: required: false show_user: false description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-entity-analytics.html#_tracer_enabled) for details. + hide_in_deployment_modes: + - agentless - name: tags type: text title: Tags @@ -220,6 +222,19 @@ streams: show_user: false description: >- Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: use_minimal_state + type: bool + title: Use minimal state + description: >- + Use the minimal-state sync implementation. Required for agentless + deployments. Uses reduced local storage and supports ES-backed state. + multi: false + required: false + show_user: false + default: true + hide_in_deployment_modes: + - default + - agentless elasticsearch: dynamic_dataset: true dynamic_namespace: true diff --git a/packages/entityanalytics_entra_id/manifest.yml b/packages/entityanalytics_entra_id/manifest.yml index cc231e30b19..67ceaa46aae 100644 --- a/packages/entityanalytics_entra_id/manifest.yml +++ b/packages/entityanalytics_entra_id/manifest.yml @@ -1,7 +1,7 @@ -format_version: "3.0.2" +format_version: "3.3.2" name: entityanalytics_entra_id title: "Microsoft Entra ID Entity Analytics" -version: "1.11.0" +version: "1.12.0" description: "Collect identities from Microsoft Entra ID (formerly Azure Active Directory) with Elastic Agent." type: integration categories: @@ -11,7 +11,7 @@ categories: - advanced_analytics_ueba conditions: kibana: - version: "^9.4.0" + version: "^9.5.0" elastic: subscription: "basic" icons: @@ -23,6 +23,15 @@ policy_templates: - name: entity title: Identities description: Collect identities. + deployment_modes: + default: + enabled: true + agentless: + enabled: true + release: beta + organization: security + division: engineering + team: security-service-integrations inputs: - type: entity-analytics title: Collect identities diff --git a/packages/entityanalytics_okta/changelog.yml b/packages/entityanalytics_okta/changelog.yml index 0c45b6fa392..52a8c67ad8e 100644 --- a/packages/entityanalytics_okta/changelog.yml +++ b/packages/entityanalytics_okta/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.5.0" + changes: + - description: Add agentless deployment mode and minimal-state sync support. + type: enhancement + link: https://github.com/elastic/integrations/pull/20212 - version: "3.4.0" changes: - description: Add the Advanced Analytics (UEBA) subcategory diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/scripts/env.txt b/packages/entityanalytics_okta/data_stream/entity/_dev/test/scripts/env.txt new file mode 100644 index 00000000000..72905d1ece6 --- /dev/null +++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/scripts/env.txt @@ -0,0 +1,10 @@ +[!exec:echo] skip 'Skipping test requiring absent echo command' + +exec echo ${PACKAGE_NAME} +stdout '^entityanalytics_okta$' + +exec echo ${DATA_STREAM} +stdout '^entity$' + +exec echo ${CURRENT_VERSION} +stdout '^[0-9]+\.[0-9]+\.[0-9]+$' diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-default-config.yml b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-default-config.yml index c0a19d3f519..370b348b5f8 100644 --- a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-default-config.yml +++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-default-config.yml @@ -5,6 +5,7 @@ data_stream: okta_domain: trial-xxxxxxx-admin.okta.com okta_token: xxxx dataset: all + use_minimal_state: false preserve_duplicate_custom_fields: true preserve_original_event: true enable_request_tracer: true diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-devices-config.yml b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-devices-config.yml index 8848709924a..44733b7de5a 100644 --- a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-devices-config.yml +++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-devices-config.yml @@ -6,6 +6,7 @@ data_stream: okta_token: xxxx dataset: users enrich_user_devices: true + use_minimal_state: false preserve_duplicate_custom_fields: true preserve_original_event: true enable_request_tracer: true diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-factors-config.yml b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-factors-config.yml index 4138210ecd6..11bd07e9cdb 100644 --- a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-factors-config.yml +++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-factors-config.yml @@ -6,6 +6,7 @@ data_stream: okta_token: xxxx dataset: users enrich_user_factors: true + use_minimal_state: false preserve_duplicate_custom_fields: true preserve_original_event: true enable_request_tracer: true diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-perms-config.yml b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-perms-config.yml index 25dd55ce7b6..60b5c546792 100644 --- a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-perms-config.yml +++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-perms-config.yml @@ -6,6 +6,7 @@ data_stream: okta_token: xxxx dataset: users enrich_user_perms: true + use_minimal_state: false preserve_duplicate_custom_fields: true preserve_original_event: true enable_request_tracer: true diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-roles-config.yml b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-roles-config.yml index bacebe6c72a..1b3dab2c9a0 100644 --- a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-roles-config.yml +++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-roles-config.yml @@ -6,6 +6,7 @@ data_stream: okta_token: xxxx dataset: all enrich_user_roles: true + use_minimal_state: false preserve_duplicate_custom_fields: true preserve_original_event: true enable_request_tracer: true diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-supervises-config.yml b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-supervises-config.yml index a9f426187de..c7711a48fdd 100644 --- a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-supervises-config.yml +++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-enrich-supervises-config.yml @@ -6,6 +6,7 @@ data_stream: okta_token: xxxx dataset: users enrich_user_supervises: true + use_minimal_state: false preserve_duplicate_custom_fields: true preserve_original_event: true enable_request_tracer: true diff --git a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-oauth2-oin-config.yml b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-oauth2-oin-config.yml index 1c505ad0cd0..69100c6cb88 100644 --- a/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-oauth2-oin-config.yml +++ b/packages/entityanalytics_okta/data_stream/entity/_dev/test/system/test-oauth2-oin-config.yml @@ -10,6 +10,7 @@ data_stream: - okta.users.read dataset: users enrich_user_roles: false + use_minimal_state: false preserve_duplicate_custom_fields: true preserve_original_event: true enable_request_tracer: true diff --git a/packages/entityanalytics_okta/data_stream/entity/agent/stream/entity-analytics.yml.hbs b/packages/entityanalytics_okta/data_stream/entity/agent/stream/entity-analytics.yml.hbs index eb9b2f26b9a..763cae1c8e9 100644 --- a/packages/entityanalytics_okta/data_stream/entity/agent/stream/entity-analytics.yml.hbs +++ b/packages/entityanalytics_okta/data_stream/entity/agent/stream/entity-analytics.yml.hbs @@ -1,4 +1,7 @@ provider: okta +{{#if use_minimal_state}} +use_minimal_state: true +{{/if}} tracer.enabled: {{enable_request_tracer}} tracer.filename: "../../logs/okta/http-request-trace-*.ndjson" tracer.maxbackups: 5 diff --git a/packages/entityanalytics_okta/data_stream/entity/manifest.yml b/packages/entityanalytics_okta/data_stream/entity/manifest.yml index 0387da55410..5bca463b8a1 100644 --- a/packages/entityanalytics_okta/data_stream/entity/manifest.yml +++ b/packages/entityanalytics_okta/data_stream/entity/manifest.yml @@ -84,6 +84,8 @@ streams: The path to the file containing the private JSON Web Key (JWK) of your Okta service application. This is only required for OAuth2 authentication. Do not use this field if you are using API Key based authentication. Only one of JWK JSON, PEM Encoded Key or JWK File may be used. + hide_in_deployment_modes: + - agentless - name: okta_scopes type: text title: Okta Scopes @@ -256,6 +258,8 @@ streams: required: false show_user: false description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_filename) for details. + hide_in_deployment_modes: + - agentless - name: tags type: text title: Tags @@ -289,6 +293,19 @@ streams: show_user: false description: >- Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: use_minimal_state + type: bool + title: Use minimal state + description: >- + Use the minimal-state sync implementation. Required for agentless + deployments. Uses reduced local storage and supports ES-backed state. + multi: false + required: false + show_user: false + default: true + hide_in_deployment_modes: + - default + - agentless elasticsearch: dynamic_dataset: true dynamic_namespace: true diff --git a/packages/entityanalytics_okta/manifest.yml b/packages/entityanalytics_okta/manifest.yml index 3f6f21d6113..b86119cdcb1 100644 --- a/packages/entityanalytics_okta/manifest.yml +++ b/packages/entityanalytics_okta/manifest.yml @@ -1,7 +1,7 @@ -format_version: "3.0.2" +format_version: "3.3.2" name: entityanalytics_okta title: Okta Entity Analytics -version: "3.4.0" +version: "3.5.0" description: "Collect Identities from Okta with Elastic Agent." type: integration categories: @@ -9,7 +9,7 @@ categories: - advanced_analytics_ueba conditions: kibana: - version: "^9.4.0" + version: "^9.5.0" elastic: subscription: "basic" screenshots: @@ -31,6 +31,15 @@ policy_templates: - name: entity title: Identities description: Collect identities. + deployment_modes: + default: + enabled: true + agentless: + enabled: true + release: beta + organization: security + division: engineering + team: security-service-integrations inputs: - type: entity-analytics title: Collect identities