diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index ad2c5b53ff4..1b452d0efdd 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.37.1" + changes: + - description: Reduce degraded documents in azure.auditlogs by raising the keyword `ignore_above` limit to `8191` on `result_description`, `result_reason`, `additional_details.value`, and `modified_properties` leaf fields; fix the ingest pipeline to normalize the top-level `resultDescription` field to snake_case so it is correctly mapped; declare the missing `properties.result_description` field mapping as well. + type: bugfix + link: https://github.com/elastic/integrations/pull/20132 - version: "1.37.0" changes: - description: Add azure.aadgraphactivitylogs dataset to ingest legacy Azure AD Graph (graph.windows.net) AzureADGraphActivityLogs diagnostic events. diff --git a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-result-description.log b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-result-description.log new file mode 100644 index 00000000000..a00e9b171dc --- /dev/null +++ b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-result-description.log @@ -0,0 +1,2 @@ +{ "time": "2022-01-22T18:15:02.3875429Z", "resourceId": "/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam", "operationName": "Update service principal", "operationVersion": "1.0", "category": "AuditLogs", "tenantId": "4bbb79f7-5724-4c9e-95f3-de075f6ec090", "resultSignature": "None", "resultDescription": "User policy updated by administrator", "durationMs": 0, "callerIpAddress": "::2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", "correlationId": "87979703-118b-498f-99c2-ccd1a56f1a5a", "identity": "Managed Service Identity", "Level": 4, "properties": {"id":"Directory_87979703-118b-498f-99c2-ccd1a56f1a5a_ULAYA_144938566","category":"ApplicationManagement","correlationId":"87979703-118b-498f-99c2-ccd1a56f1a5a","result":"success","resultReason":"","activityDisplayName":"Update service principal","activityDateTime":"2022-01-22T18:15:02.3875429+00:00","loggedByService":"Core Directory","operationType":"Update","userAgent":null,"initiatedBy":{"app":{"appId":null,"displayName":"Managed Service Identity","servicePrincipalId":"b9814691-9ca1-4e55-a1ac-8ef5dd010ec0","servicePrincipalName":null}},"targetResources":[{"id":"a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42","displayName":"billing-test-wus","type":"ServicePrincipal","modifiedProperties":[{"displayName":"TargetId.ServicePrincipalNames","oldValue":null,"newValue":"\"a70a7931-c387-4dce-9f35-fbf95bdcc91e;https://identity.azure.net/N8CUySpCeRFU3iB/PEuFlON4zd8+n8d3qgzrF1MviSY=\""}],"administrativeUnits":[]}],"additionalDetails":[{"key":"User-Agent","value":"Microsoft Azure Graph Client Library 2.1.17-internal"},{"key":"AppId","value":"a70a7931-c387-4dce-9f35-fbf95bdcc91e"}]}} +{ "time": "2022-01-22T18:15:02.3875429Z", "resourceId": "/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam", "operationName": "Update policy", "operationVersion": "1.0", "category": "AuditLogs", "tenantId": "4bbb79f7-5724-4c9e-95f3-de075f6ec090", "resultSignature": "None", "durationMs": 0, "callerIpAddress": "::2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", "correlationId": "87979703-118b-498f-99c2-ccd1a56f1a5a", "identity": "Managed Service Identity", "Level": 4, "properties": {"id":"Directory_87979703-118b-498f-99c2-ccd1a56f1a5a_ULAYA_144938567","category":"Policy","correlationId":"87979703-118b-498f-99c2-ccd1a56f1a5a","result":"success","resultReason":"","resultDescription":"Conditional access policy was updated.","activityDisplayName":"Update policy","activityDateTime":"2022-01-22T18:15:02.3875429+00:00","loggedByService":"Core Directory","operationType":"Update","userAgent":null,"initiatedBy":{"app":{"appId":null,"displayName":"Managed Service Identity","servicePrincipalId":"b9814691-9ca1-4e55-a1ac-8ef5dd010ec0","servicePrincipalName":null}},"targetResources":[{"id":"a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42","displayName":"TestPolicy","type":"Policy","modifiedProperties":[{"displayName":"PolicyName","oldValue":"\"OldPolicyName\"","newValue":"\"NewPolicyName\""}],"administrativeUnits":[]}],"additionalDetails":[]}} diff --git a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-result-description.log-expected.json b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-result-description.log-expected.json new file mode 100644 index 00000000000..5c9c08ee065 --- /dev/null +++ b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-result-description.log-expected.json @@ -0,0 +1,164 @@ +{ + "expected": [ + { + "@timestamp": "2022-01-22T18:15:02.387Z", + "azure": { + "auditlogs": { + "category": "AuditLogs", + "identity": "Managed Service Identity", + "level": 4, + "operation_name": "Update service principal", + "operation_version": "1.0", + "properties": { + "activity_datetime": "2022-01-22T18:15:02.3875429+00:00", + "activity_display_name": "Update service principal", + "additional_details": [ + { + "key": "User-Agent", + "value": "Microsoft Azure Graph Client Library 2.1.17-internal" + }, + { + "key": "AppId", + "value": "a70a7931-c387-4dce-9f35-fbf95bdcc91e" + } + ], + "category": "ApplicationManagement", + "correlation_id": "87979703-118b-498f-99c2-ccd1a56f1a5a", + "id": "Directory_87979703-118b-498f-99c2-ccd1a56f1a5a_ULAYA_144938566", + "initiated_by": { + "app": { + "displayName": "Managed Service Identity", + "servicePrincipalId": "b9814691-9ca1-4e55-a1ac-8ef5dd010ec0" + } + }, + "logged_by_service": "Core Directory", + "operation_type": "Update", + "result_reason": "", + "target_resources": { + "0": { + "display_name": "billing-test-wus", + "id": "a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42", + "modified_properties": { + "0": { + "display_name": "TargetId.ServicePrincipalNames", + "new_value": "\"a70a7931-c387-4dce-9f35-fbf95bdcc91e;https://identity.azure.net/N8CUySpCeRFU3iB/PEuFlON4zd8+n8d3qgzrF1MviSY=\"" + } + }, + "type": "ServicePrincipal" + } + } + }, + "result_description": "User policy updated by administrator", + "result_signature": "None" + }, + "correlation_id": "87979703-118b-498f-99c2-ccd1a56f1a5a", + "resource": { + "id": "/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam", + "provider": "Microsoft.aadiam" + }, + "tenant_id": "4bbb79f7-5724-4c9e-95f3-de075f6ec090" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Update service principal", + "duration": 0, + "kind": "event", + "original": "{ \"time\": \"2022-01-22T18:15:02.3875429Z\", \"resourceId\": \"/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam\", \"operationName\": \"Update service principal\", \"operationVersion\": \"1.0\", \"category\": \"AuditLogs\", \"tenantId\": \"4bbb79f7-5724-4c9e-95f3-de075f6ec090\", \"resultSignature\": \"None\", \"resultDescription\": \"User policy updated by administrator\", \"durationMs\": 0, \"callerIpAddress\": \"::2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6\", \"correlationId\": \"87979703-118b-498f-99c2-ccd1a56f1a5a\", \"identity\": \"Managed Service Identity\", \"Level\": 4, \"properties\": {\"id\":\"Directory_87979703-118b-498f-99c2-ccd1a56f1a5a_ULAYA_144938566\",\"category\":\"ApplicationManagement\",\"correlationId\":\"87979703-118b-498f-99c2-ccd1a56f1a5a\",\"result\":\"success\",\"resultReason\":\"\",\"activityDisplayName\":\"Update service principal\",\"activityDateTime\":\"2022-01-22T18:15:02.3875429+00:00\",\"loggedByService\":\"Core Directory\",\"operationType\":\"Update\",\"userAgent\":null,\"initiatedBy\":{\"app\":{\"appId\":null,\"displayName\":\"Managed Service Identity\",\"servicePrincipalId\":\"b9814691-9ca1-4e55-a1ac-8ef5dd010ec0\",\"servicePrincipalName\":null}},\"targetResources\":[{\"id\":\"a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42\",\"displayName\":\"billing-test-wus\",\"type\":\"ServicePrincipal\",\"modifiedProperties\":[{\"displayName\":\"TargetId.ServicePrincipalNames\",\"oldValue\":null,\"newValue\":\"\\\"a70a7931-c387-4dce-9f35-fbf95bdcc91e;https://identity.azure.net/N8CUySpCeRFU3iB/PEuFlON4zd8+n8d3qgzrF1MviSY=\\\"\"}],\"administrativeUnits\":[]}],\"additionalDetails\":[{\"key\":\"User-Agent\",\"value\":\"Microsoft Azure Graph Client Library 2.1.17-internal\"},{\"key\":\"AppId\",\"value\":\"a70a7931-c387-4dce-9f35-fbf95bdcc91e\"}]}}", + "outcome": "success" + }, + "related": { + "entity": [ + "b9814691-9ca1-4e55-a1ac-8ef5dd010ec0", + "a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42" + ] + }, + "source": { + "address": "::2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-01-22T18:15:02.387Z", + "azure": { + "auditlogs": { + "category": "AuditLogs", + "identity": "Managed Service Identity", + "level": 4, + "operation_name": "Update policy", + "operation_version": "1.0", + "properties": { + "activity_datetime": "2022-01-22T18:15:02.3875429+00:00", + "activity_display_name": "Update policy", + "additional_details": [], + "category": "Policy", + "correlation_id": "87979703-118b-498f-99c2-ccd1a56f1a5a", + "id": "Directory_87979703-118b-498f-99c2-ccd1a56f1a5a_ULAYA_144938567", + "initiated_by": { + "app": { + "displayName": "Managed Service Identity", + "servicePrincipalId": "b9814691-9ca1-4e55-a1ac-8ef5dd010ec0" + } + }, + "logged_by_service": "Core Directory", + "operation_type": "Update", + "result_description": "Conditional access policy was updated.", + "result_reason": "", + "target_resources": { + "0": { + "display_name": "TestPolicy", + "id": "a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42", + "modified_properties": { + "0": { + "display_name": "PolicyName", + "new_value": "\"NewPolicyName\"", + "old_value": "\"OldPolicyName\"" + } + }, + "type": "Policy" + } + } + }, + "result_signature": "None" + }, + "correlation_id": "87979703-118b-498f-99c2-ccd1a56f1a5a", + "resource": { + "id": "/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam", + "provider": "Microsoft.aadiam" + }, + "tenant_id": "4bbb79f7-5724-4c9e-95f3-de075f6ec090" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Update policy", + "duration": 0, + "kind": "event", + "original": "{ \"time\": \"2022-01-22T18:15:02.3875429Z\", \"resourceId\": \"/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam\", \"operationName\": \"Update policy\", \"operationVersion\": \"1.0\", \"category\": \"AuditLogs\", \"tenantId\": \"4bbb79f7-5724-4c9e-95f3-de075f6ec090\", \"resultSignature\": \"None\", \"durationMs\": 0, \"callerIpAddress\": \"::2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6\", \"correlationId\": \"87979703-118b-498f-99c2-ccd1a56f1a5a\", \"identity\": \"Managed Service Identity\", \"Level\": 4, \"properties\": {\"id\":\"Directory_87979703-118b-498f-99c2-ccd1a56f1a5a_ULAYA_144938567\",\"category\":\"Policy\",\"correlationId\":\"87979703-118b-498f-99c2-ccd1a56f1a5a\",\"result\":\"success\",\"resultReason\":\"\",\"resultDescription\":\"Conditional access policy was updated.\",\"activityDisplayName\":\"Update policy\",\"activityDateTime\":\"2022-01-22T18:15:02.3875429+00:00\",\"loggedByService\":\"Core Directory\",\"operationType\":\"Update\",\"userAgent\":null,\"initiatedBy\":{\"app\":{\"appId\":null,\"displayName\":\"Managed Service Identity\",\"servicePrincipalId\":\"b9814691-9ca1-4e55-a1ac-8ef5dd010ec0\",\"servicePrincipalName\":null}},\"targetResources\":[{\"id\":\"a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42\",\"displayName\":\"TestPolicy\",\"type\":\"Policy\",\"modifiedProperties\":[{\"displayName\":\"PolicyName\",\"oldValue\":\"\\\"OldPolicyName\\\"\",\"newValue\":\"\\\"NewPolicyName\\\"\"}],\"administrativeUnits\":[]}],\"additionalDetails\":[]}}", + "outcome": "success" + }, + "related": { + "entity": [ + "b9814691-9ca1-4e55-a1ac-8ef5dd010ec0", + "a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42" + ] + }, + "source": { + "address": "::2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} diff --git a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml index f11d67265e9..5f7db6f9d76 100644 --- a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml @@ -96,6 +96,10 @@ processors: field: azure.auditlogs.resultSignature target_field: azure.auditlogs.result_signature ignore_missing: true + - rename: + field: azure.auditlogs.resultDescription + target_field: azure.auditlogs.result_description + ignore_missing: true - rename: field: azure.auditlogs.operationVersion target_field: azure.auditlogs.operation_version diff --git a/packages/azure/data_stream/auditlogs/fields/fields.yml b/packages/azure/data_stream/auditlogs/fields/fields.yml index 86007e6ae5e..fd637420236 100644 --- a/packages/azure/data_stream/auditlogs/fields/fields.yml +++ b/packages/azure/data_stream/auditlogs/fields/fields.yml @@ -27,6 +27,7 @@ Result signature - name: result_description type: keyword + ignore_above: 8191 description: | Result description - name: level @@ -45,8 +46,14 @@ Activity display name - name: result_reason type: keyword + ignore_above: 8191 description: | Reason for the log result + - name: result_description + type: keyword + ignore_above: 8191 + description: | + Result description - name: correlation_id type: keyword description: | @@ -94,21 +101,14 @@ type: keyword description: | User principal name - - name: modified_properties.* - type: group - fields: - - name: new_value - type: keyword - description: | - New value - - name: display_name - type: keyword - description: | - Display value - - name: old_value - type: keyword - description: | - Old value + - name: modified_properties.*.* + type: object + object_type: keyword + object_type_mapping_type: '*' + ignore_above: 8191 + subobjects: false + description: | + Modified properties (for example, old_value, new_value, display_name). - name: initiated_by type: group fields: @@ -165,6 +165,7 @@ description: Additional details key - name: value type: keyword + ignore_above: 8191 description: Additional details value - name: authentication_protocol type: keyword diff --git a/packages/azure/docs/adlogs.md b/packages/azure/docs/adlogs.md index b576bb9ec92..7d37a3a1aae 100644 --- a/packages/azure/docs/adlogs.md +++ b/packages/azure/docs/adlogs.md @@ -810,13 +810,12 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | azure.auditlogs.properties.logged_by_service | Logged by service | keyword | | azure.auditlogs.properties.operation_type | Operation type | keyword | | azure.auditlogs.properties.result | Log result | keyword | +| azure.auditlogs.properties.result_description | Result description | keyword | | azure.auditlogs.properties.result_reason | Reason for the log result | keyword | | azure.auditlogs.properties.target_resources.\*.display_name | Display name | keyword | | azure.auditlogs.properties.target_resources.\*.id | ID | keyword | | azure.auditlogs.properties.target_resources.\*.ip_address | ip Address | keyword | -| azure.auditlogs.properties.target_resources.\*.modified_properties.\*.display_name | Display value | keyword | -| azure.auditlogs.properties.target_resources.\*.modified_properties.\*.new_value | New value | keyword | -| azure.auditlogs.properties.target_resources.\*.modified_properties.\*.old_value | Old value | keyword | +| azure.auditlogs.properties.target_resources.\*.modified_properties.\*.\* | Modified properties (for example, old_value, new_value, display_name). | object | | azure.auditlogs.properties.target_resources.\*.type | Type | keyword | | azure.auditlogs.properties.target_resources.\*.user_principal_name | User principal name | keyword | | azure.auditlogs.result_description | Result description | keyword | diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index 4d36c4ee9d2..ef84192596d 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.37.0" +version: "1.37.1" description: This Elastic integration collects logs from Azure type: integration icons: