From 6c59086a2b9cc95711977938b91d64a09c8a9a98 Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Thu, 14 May 2026 13:25:37 -0500 Subject: [PATCH 01/10] [OCSF] Zeek/Corelight pipeline Add OCSF v1.5.0 normalization for Zeek/Corelight logs, covering 7 log types across 5 OCSF classes (Detection Finding, Network Activity, HTTP Activity, DNS Activity, File Hosting Activity). Co-Authored-By: Claude Sonnet 4.6 --- zeek/assets/logs/zeek.yaml | 2462 +++++++++++++++++++++++++++++- zeek/assets/logs/zeek_tests.yaml | 1109 +++++++++++++- 2 files changed, 3482 insertions(+), 89 deletions(-) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index 528fba092c81b..0236b4e372be0 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -390,6 +390,441 @@ facets: path: zeek.x509_total source: log type: double + - groups: + - OCSF + name: Activity ID + path: ocsf.activity_id + source: log + - groups: + - OCSF + name: Activity Name + path: ocsf.activity_name + source: log + - groups: + - OCSF + name: Category UID + path: ocsf.category_uid + source: log + - groups: + - OCSF + name: Category Name + path: ocsf.category_name + source: log + - groups: + - OCSF + name: Class UID + path: ocsf.class_uid + source: log + - groups: + - OCSF + name: Class Name + path: ocsf.class_name + source: log + - groups: + - OCSF + name: Severity + path: ocsf.severity + source: log + - groups: + - OCSF + name: Severity ID + path: ocsf.severity_id + source: log + - groups: + - OCSF + name: Status + path: ocsf.status + source: log + - groups: + - OCSF + name: Status ID + path: ocsf.status_id + source: log + - groups: + - OCSF + name: Source Endpoint IP + path: ocsf.src_endpoint.ip + source: log + - groups: + - OCSF + name: Destination Endpoint IP + path: ocsf.dst_endpoint.ip + source: log + - groups: + - OCSF + name: Event Code + path: ocsf.metadata.event_code + source: log + - groups: + - OCSF + name: Product Name + path: ocsf.metadata.product.name + source: log + - groups: + - OCSF + name: Product Vendor Name + path: ocsf.metadata.product.vendor_name + source: log + - groups: + - OCSF + name: Finding Title + path: ocsf.finding_info.title + source: log + - groups: + - OCSF + name: Finding UID + path: ocsf.finding_info.uid + source: log + - groups: + - OCSF + name: HTTP Method + path: ocsf.http_request.http_method + source: log + - groups: + - OCSF + name: URL String + path: ocsf.http_request.url.url_string + source: log + - groups: + - OCSF + name: TLS Version + path: ocsf.tls.version + source: log + - groups: + - OCSF + name: File Name + path: ocsf.file.name + source: log + - groups: + - OCSF + name: File MIME Type + path: ocsf.file.mime_type + source: log + - groups: + - OCSF + name: DNS Query Hostname + path: ocsf.query.hostname + source: log + - groups: + - OCSF + name: Actor Session UID + path: ocsf.actor.session.uid + source: log + - groups: + - OCSF + name: Confidence + path: ocsf.confidence + source: log + - groups: + - OCSF + name: Confidence ID + path: ocsf.confidence_id + source: log + - groups: + - OCSF + name: Connection Info Boundary + path: ocsf.connection_info.boundary + source: log + - groups: + - OCSF + name: Connection Info Boundary ID + path: ocsf.connection_info.boundary_id + source: log + - groups: + - OCSF + name: Connection Info Community UID + path: ocsf.connection_info.community_uid + source: log + - groups: + - OCSF + name: Connection Info Direction + path: ocsf.connection_info.direction + source: log + - groups: + - OCSF + name: Connection Info Direction ID + path: ocsf.connection_info.direction_id + source: log + - groups: + - OCSF + name: Connection Info Flag History + path: ocsf.connection_info.flag_history + source: log + - groups: + - OCSF + name: Connection Info Protocol Name + path: ocsf.connection_info.protocol_name + source: log + - groups: + - OCSF + name: Connection Info Protocol Ver + path: ocsf.connection_info.protocol_ver + source: log + - groups: + - OCSF + name: Connection Info UID + path: ocsf.connection_info.uid + source: log + - groups: + - OCSF + name: Device IP + path: ocsf.device.ip + source: log + - groups: + - OCSF + name: Dst Endpoint Hostname + path: ocsf.dst_endpoint.hostname + source: log + - groups: + - OCSF + name: Dst Endpoint Port + path: ocsf.dst_endpoint.port + source: log + - groups: + - OCSF + name: Duration + path: ocsf.duration + source: log + - groups: + - OCSF + name: Evidences + path: ocsf.evidences + source: log + - groups: + - OCSF + name: File Hashes + path: ocsf.file.hashes + source: log + - groups: + - OCSF + name: File Size + path: ocsf.file.size + source: log + - groups: + - OCSF + name: File Type ID + path: ocsf.file.type_id + source: log + - groups: + - OCSF + name: File UID + path: ocsf.file.uid + source: log + - groups: + - OCSF + name: Finding Info Analytic Name + path: ocsf.finding_info.analytic.name + source: log + - groups: + - OCSF + name: Finding Info Analytic Type + path: ocsf.finding_info.analytic.type + source: log + - groups: + - OCSF + name: Finding Info Analytic Type ID + path: ocsf.finding_info.analytic.type_id + source: log + - groups: + - OCSF + name: Finding Info Analytic UID + path: ocsf.finding_info.analytic.uid + source: log + - groups: + - OCSF + name: Finding Info UID Alt + path: ocsf.finding_info.uid_alt + source: log + - groups: + - OCSF + name: HTTP Request Body Length + path: ocsf.http_request.body_length + source: log + - groups: + - OCSF + name: HTTP Request UID + path: ocsf.http_request.uid + source: log + - groups: + - OCSF + name: HTTP Request URL Hostname + path: ocsf.http_request.url.hostname + source: log + - groups: + - OCSF + name: HTTP Request URL Path + path: ocsf.http_request.url.path + source: log + - groups: + - OCSF + name: HTTP Request URL Port + path: ocsf.http_request.url.port + source: log + - groups: + - OCSF + name: HTTP Request User Agent + path: ocsf.http_request.user_agent + source: log + - groups: + - OCSF + name: HTTP Request Version + path: ocsf.http_request.version + source: log + - groups: + - OCSF + name: HTTP Response Body Length + path: ocsf.http_response.body_length + source: log + - groups: + - OCSF + name: HTTP Response Code + path: ocsf.http_response.code + source: log + - groups: + - OCSF + name: HTTP Response Message + path: ocsf.http_response.message + source: log + - groups: + - OCSF + name: Is Alert + path: ocsf.is_alert + source: log + - groups: + - OCSF + name: Message + path: ocsf.message + source: log + - groups: + - OCSF + name: Metadata Correlation UID + path: ocsf.metadata.correlation_uid + source: log + - groups: + - OCSF + name: Metadata Log Provider + path: ocsf.metadata.log_provider + source: log + - groups: + - OCSF + name: Metadata Original Time + path: ocsf.metadata.original_time + source: log + - groups: + - OCSF + name: Metadata UID + path: ocsf.metadata.uid + source: log + - groups: + - OCSF + name: Query Class + path: ocsf.query.class + source: log + - groups: + - OCSF + name: Query Type + path: ocsf.query.type + source: log + - groups: + - OCSF + name: Rcode + path: ocsf.rcode + source: log + - groups: + - OCSF + name: Rcode ID + path: ocsf.rcode_id + source: log + - groups: + - OCSF + name: Src Endpoint Port + path: ocsf.src_endpoint.port + source: log + - groups: + - OCSF + name: Status Detail + path: ocsf.status_detail + source: log + - groups: + - OCSF + name: Time + path: ocsf.time + source: log + - groups: + - OCSF + name: TLS Cipher + path: ocsf.tls.cipher + source: log + - groups: + - OCSF + name: TLS JA3 Hash Algorithm + path: ocsf.tls.ja3_hash.algorithm + source: log + - groups: + - OCSF + name: TLS JA3 Hash Algorithm ID + path: ocsf.tls.ja3_hash.algorithm_id + source: log + - groups: + - OCSF + name: TLS JA3 Hash Value + path: ocsf.tls.ja3_hash.value + source: log + - groups: + - OCSF + name: TLS JA3s Hash Algorithm + path: ocsf.tls.ja3s_hash.algorithm + source: log + - groups: + - OCSF + name: TLS JA3s Hash Algorithm ID + path: ocsf.tls.ja3s_hash.algorithm_id + source: log + - groups: + - OCSF + name: TLS JA3s Hash Value + path: ocsf.tls.ja3s_hash.value + source: log + - groups: + - OCSF + name: TLS SNI + path: ocsf.tls.sni + source: log + - groups: + - OCSF + name: Traffic Bytes + path: ocsf.traffic.bytes + source: log + - groups: + - OCSF + name: Traffic Bytes In + path: ocsf.traffic.bytes_in + source: log + - groups: + - OCSF + name: Traffic Bytes Missed + path: ocsf.traffic.bytes_missed + source: log + - groups: + - OCSF + name: Traffic Bytes Out + path: ocsf.traffic.bytes_out + source: log + - groups: + - OCSF + name: Traffic Packets + path: ocsf.traffic.packets + source: log + - groups: + - OCSF + name: Traffic Packets In + path: ocsf.traffic.packets_in + source: log + - groups: + - OCSF + name: Traffic Packets Out + path: ocsf.traffic.packets_out + source: log pipeline: type: pipeline name: Zeek @@ -425,7 +860,7 @@ pipeline: sourceType: attribute target: network.client.ip targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `id.orig_p` to `network.client.port` @@ -435,7 +870,7 @@ pipeline: sourceType: attribute target: network.client.port targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `id.resp_h` to `network.destination.ip` @@ -445,7 +880,7 @@ pipeline: sourceType: attribute target: network.destination.ip targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `id.resp_p` to `network.destination.port` @@ -455,7 +890,7 @@ pipeline: sourceType: attribute target: network.destination.port targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Datetime Remapper for other than files_red logs @@ -512,7 +947,7 @@ pipeline: sourceType: attribute target: zeek.missed_bytes targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Processing for pe logs @@ -598,7 +1033,7 @@ pipeline: sourceType: attribute target: http.method targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `host` to `http.url_details.host` @@ -608,7 +1043,7 @@ pipeline: sourceType: attribute target: http.url_details.host targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `uri` to `http.url_details.path` @@ -618,7 +1053,7 @@ pipeline: sourceType: attribute target: http.url_details.path targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `referrer` to `http.referer` @@ -628,7 +1063,7 @@ pipeline: sourceType: attribute target: http.referer targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `version` to `http.version` @@ -638,7 +1073,7 @@ pipeline: sourceType: attribute target: http.version targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `user_agent` to `http.useragent` @@ -648,7 +1083,7 @@ pipeline: sourceType: attribute target: http.useragent targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `status_code` to `http.status_code` @@ -658,7 +1093,7 @@ pipeline: sourceType: attribute target: http.status_code targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Standard attribute remapping for Web Access @@ -675,7 +1110,7 @@ pipeline: sourceType: attribute target: network.bytes_read targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `resp_bytes`, `response_body_len` to `network.bytes_written` @@ -686,7 +1121,7 @@ pipeline: sourceType: attribute target: network.bytes_written targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Processing for dns logs and Standard attribute remapping for DNS @@ -719,7 +1154,7 @@ pipeline: sourceType: attribute target: dns.id targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `domain`, `query` to `dns.question.name` @@ -730,7 +1165,7 @@ pipeline: sourceType: attribute target: dns.question.name targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `qclass_name` to `dns.question.class` @@ -740,7 +1175,7 @@ pipeline: sourceType: attribute target: dns.question.class targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `qtype_name` to `dns.question.type` @@ -750,7 +1185,7 @@ pipeline: sourceType: attribute target: dns.question.type targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `rcode_name` to `dns.flags.rcode` @@ -760,7 +1195,7 @@ pipeline: sourceType: attribute target: dns.flags.rcode targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `answers` to `dns.answer.name` @@ -770,7 +1205,7 @@ pipeline: sourceType: attribute target: dns.answer.name targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Standard attribute remapping for user attributes @@ -865,7 +1300,7 @@ pipeline: sourceType: attribute target: zeek.duration_sec targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Remap attributes for ssh logs @@ -929,7 +1364,7 @@ pipeline: sourceType: attribute target: zeek.note targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Remap attributes for files logs @@ -955,7 +1390,7 @@ pipeline: sourceType: attribute target: zeek.missing_bytes targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `seen_bytes` to `zeek.seen_bytes` @@ -965,7 +1400,7 @@ pipeline: sourceType: attribute target: zeek.seen_bytes targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `total_bytes` to `zeek.total_bytes` @@ -975,7 +1410,7 @@ pipeline: sourceType: attribute target: zeek.total_bytes targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `network.client.ip`, `rx_hosts` to `orig_host` @@ -1061,7 +1496,7 @@ pipeline: sourceType: attribute target: zeek.proto targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Remapping datared log attributes @@ -1233,7 +1668,7 @@ pipeline: sourceType: attribute target: network.client.port targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: status-remapper name: Define `severity`, `syslog.severity` as the official status of the log @@ -1241,3 +1676,1976 @@ pipeline: sources: - severity - syslog.severity + - type: pipeline + name: OCSF pre transformations + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(files OR files_red OR notice OR suricata_corelight OR conn OR conn_long OR conn_red OR ssl OR ssl_red OR weird_red OR http OR http_red OR dns OR dns_red)" + processors: + - type: string-builder-processor + name: Add product name + enabled: true + template: Zeek + target: ocsf.metadata.product.name + replaceMissing: false + - type: string-builder-processor + name: Add product vendor name + enabled: true + template: Corelight + target: ocsf.metadata.product.vendor_name + replaceMissing: false + - type: grok-parser + name: Parse `ts` to epoch milliseconds + enabled: true + source: ts + samples: + - "2023-12-12T05:52:32.763303Z" + grok: + supportRules: "" + matchRules: 'parse_ts %{date("yyyy-MM-dd''T''HH:mm:ss.SSSSSSZ"):_time_ms}' + - type: attribute-remapper + name: Map `_time_ms` to `ocsf.time` + enabled: true + sources: + - _time_ms + sourceType: attribute + target: ocsf.time + targetType: attribute + preserveSource: false + - type: pipeline + name: OCSF sub pipeline for class Detection Finding [2004] - Notice + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:notice" + processors: + - type: string-builder-processor + name: Set is_alert to boolean true + enabled: true + template: "true" + target: _is_alert_str + replaceMissing: false + - type: grok-parser + name: Convert _is_alert_str to boolean ocsf.is_alert + enabled: true + source: _is_alert_str + samples: + - "true" + grok: + supportRules: "" + matchRules: "to_bool %{boolean(\"true\",\"false\"):ocsf.is_alert}" + - type: attribute-remapper + name: Map `id.orig_h` to `ocsf.evidence.src_endpoint.ip` + enabled: true + sources: + - id.orig_h + sourceType: attribute + target: ocsf.evidence.src_endpoint.ip + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.orig_p` to `ocsf.evidence.src_endpoint.port` + enabled: true + sources: + - id.orig_p + sourceType: attribute + target: ocsf.evidence.src_endpoint.port + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.resp_h` to `ocsf.evidence.dst_endpoint.ip` + enabled: true + sources: + - id.resp_h + sourceType: attribute + target: ocsf.evidence.dst_endpoint.ip + targetType: attribute + preserveSource: true + - type: attribute-remapper + name: Map `id.resp_p` to `ocsf.evidence.dst_endpoint.port` + enabled: true + sources: + - id.resp_p + sourceType: attribute + target: ocsf.evidence.dst_endpoint.port + targetType: attribute + preserveSource: true + - type: array-processor + name: Move ocsf.evidence into ocsf.evidences array + enabled: true + operation: + source: ocsf.evidence + target: ocsf.evidences + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 2004 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "*" + name: Create + id: 1 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.category_uid + categories: + - filter: + query: "*" + name: Findings + id: 2 + targets: + name: ocsf.category_name + id: ocsf.category_uid + - type: schema-remapper + name: Map `ocsf.evidences` to `ocsf.evidences` + sources: + - ocsf.evidences + target: ocsf.evidences + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `note` to `ocsf.finding_info.title` + sources: + - note + target: ocsf.finding_info.title + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.finding_info.uid` + sources: + - uid + target: ocsf.finding_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `note` to `ocsf.finding_info.uid` + sources: + - note + target: ocsf.finding_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.is_alert` to `ocsf.is_alert` + sources: + - ocsf.is_alert + target: ocsf.is_alert + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `note` to `ocsf.metadata.event_code` + sources: + - note + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "@severity.name:informational" + name: Informational + id: 1 + - filter: + query: "@severity.name:low" + name: Low + id: 2 + - filter: + query: "@severity.name:medium" + name: Medium + id: 3 + - filter: + query: "@severity.name:high" + name: High + id: 4 + - filter: + query: "@severity.name:critical" + name: Critical + id: 5 + - filter: + query: "@severity.name:*" + name: Other + id: 99 + targets: + name: ocsf.severity + id: ocsf.severity_id + fallback: + values: + ocsf.severity: Other + ocsf.severity_id: "99" + sources: + ocsf.severity: + - severity.name + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: New + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: Detection Finding + classUid: 2004 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Detection Finding [2004] - Suricata + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:suricata_corelight" + processors: + - type: string-builder-processor + name: Override product name for Suricata + enabled: true + template: Suricata + target: ocsf.metadata.product.name + replaceMissing: false + - type: string-builder-processor + name: Set log provider + enabled: true + template: Corelight + target: ocsf.metadata.log_provider + replaceMissing: false + - type: string-builder-processor + name: Map alert.signature_id to ocsf.metadata.event_code + enabled: true + template: "%{alert.signature_id}" + target: ocsf.metadata.event_code + replaceMissing: true + - type: string-builder-processor + name: Set is_alert to boolean true + enabled: true + template: "true" + target: _is_alert_str + replaceMissing: false + - type: grok-parser + name: Convert _is_alert_str to boolean ocsf.is_alert + enabled: true + source: _is_alert_str + samples: + - "true" + grok: + supportRules: "" + matchRules: "to_bool %{boolean(\"true\",\"false\"):ocsf.is_alert}" + - type: attribute-remapper + name: Map `id.orig_h` to `ocsf.evidence.src_endpoint.ip` + enabled: true + sources: + - id.orig_h + sourceType: attribute + target: ocsf.evidence.src_endpoint.ip + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.orig_p` to `ocsf.evidence.src_endpoint.port` + enabled: true + sources: + - id.orig_p + sourceType: attribute + target: ocsf.evidence.src_endpoint.port + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.resp_h` to `ocsf.evidence.dst_endpoint.ip` + enabled: true + sources: + - id.resp_h + sourceType: attribute + target: ocsf.evidence.dst_endpoint.ip + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.resp_p` to `ocsf.evidence.dst_endpoint.port` + enabled: true + sources: + - id.resp_p + sourceType: attribute + target: ocsf.evidence.dst_endpoint.port + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Move ocsf.evidence into ocsf.evidences array + enabled: true + operation: + source: ocsf.evidence + target: ocsf.evidences + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 2004 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "*" + name: Create + id: 1 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.category_uid + categories: + - filter: + query: "*" + name: Findings + id: 2 + targets: + name: ocsf.category_name + id: ocsf.category_uid + - type: schema-category-mapper + name: ocsf.confidence_id + categories: + - filter: + query: "@alert.metadata:\"confidence:High\"" + name: High + id: 3 + - filter: + query: "@alert.metadata:\"confidence:Medium\"" + name: Medium + id: 2 + - filter: + query: "@alert.metadata:\"confidence:Low\"" + name: Low + id: 1 + targets: + name: ocsf.confidence + id: ocsf.confidence_id + - type: schema-remapper + name: Map `ocsf.evidences` to `ocsf.evidences` + sources: + - ocsf.evidences + target: ocsf.evidences + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `alert.signature` to `ocsf.finding_info.analytic.name` + sources: + - alert.signature + target: ocsf.finding_info.analytic.name + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.finding_info.analytic.type_id + categories: + - filter: + query: "*" + name: Rule + id: 1 + targets: + name: ocsf.finding_info.analytic.type + id: ocsf.finding_info.analytic.type_id + - type: schema-remapper + name: Map `alert.signature_id` to `ocsf.finding_info.analytic.uid` + sources: + - alert.signature_id + target: ocsf.finding_info.analytic.uid + preserveSource: true + overrideOnConflict: true + targetFormat: string + - type: schema-remapper + name: Map `alert.signature` to `ocsf.finding_info.title` + sources: + - alert.signature + target: ocsf.finding_info.title + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.finding_info.uid` to `ocsf.finding_info.uid` + sources: + - ocsf.finding_info.uid + target: ocsf.finding_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.finding_info.uid` + sources: + - uid + target: ocsf.finding_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `suri_id` to `ocsf.finding_info.uid_alt` + sources: + - suri_id + target: ocsf.finding_info.uid_alt + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.is_alert` to `ocsf.is_alert` + sources: + - ocsf.is_alert + target: ocsf.is_alert + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `community_id` to `ocsf.metadata.correlation_uid` + sources: + - community_id + target: ocsf.metadata.correlation_uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` + sources: + - ocsf.metadata.event_code + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.log_provider` to `ocsf.metadata.log_provider` + sources: + - ocsf.metadata.log_provider + target: ocsf.metadata.log_provider + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "@alert.severity:1" + name: High + id: 4 + - filter: + query: "@alert.severity:2" + name: Medium + id: 3 + - filter: + query: "@alert.severity:3" + name: Low + id: 2 + targets: + name: ocsf.severity + id: ocsf.severity_id + fallback: + values: + ocsf.severity: Other + ocsf.severity_id: "99" + sources: + ocsf.severity: + - alert.severity + - type: schema-remapper + name: Map `alert.action` to `ocsf.status_detail` + sources: + - alert.action + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: Detection Finding + classUid: 2004 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Network Activity [4001] - conn + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(conn OR conn_long OR conn_red)" + processors: + - type: arithmetic-processor + name: Calculate total bytes + enabled: true + expression: (orig_bytes + resp_bytes) + target: _total_bytes + isReplaceMissing: false + - type: arithmetic-processor + name: Calculate total packets + enabled: true + expression: (orig_pkts + resp_pkts) + target: _total_packets + isReplaceMissing: false + - type: arithmetic-processor + name: Convert duration to milliseconds + enabled: true + expression: duration * 1000 + target: _duration_ms + isReplaceMissing: false + - type: schema-processor + name: Apply OCSF schema for 4001 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@conn_state:(SF OR RSTO OR RSTR OR RSTRH OR SH OR SHR)" + name: Close + id: 2 + - filter: + query: "@conn_state:(S0 OR RSTOS0)" + name: Fail + id: 4 + - filter: + query: "@conn_state:REJ" + name: Refuse + id: 5 + - filter: + query: "@conn_state:(OTH OR S1 OR S2 OR S3)" + name: Traffic + id: 6 + - filter: + query: "@conn_state:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - conn_state + - type: schema-category-mapper + name: ocsf.connection_info.boundary_id + categories: + - filter: + query: "@local_orig:true @local_resp:true" + name: Localhost + id: 1 + - filter: + query: "(@local_orig:true @local_resp:false) OR (@local_orig:false @local_resp:true)" + name: External + id: 3 + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.connection_info.boundary + id: ocsf.connection_info.boundary_id + - type: schema-remapper + name: Map `community_id` to `ocsf.connection_info.community_uid` + sources: + - community_id + target: ocsf.connection_info.community_uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.connection_info.direction_id + categories: + - filter: + query: "@local_orig:true @local_resp:false" + name: Outbound + id: 2 + - filter: + query: "@local_orig:false @local_resp:true" + name: Inbound + id: 1 + - filter: + query: "@local_orig:true @local_resp:true" + name: Lateral + id: 3 + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.connection_info.direction + id: ocsf.connection_info.direction_id + - type: schema-remapper + name: Map `history` to `ocsf.connection_info.flag_history` + sources: + - history + target: ocsf.connection_info.flag_history + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `proto` to `ocsf.connection_info.protocol_name` + sources: + - proto + target: ocsf.connection_info.protocol_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.connection_info.uid` + sources: + - uid + target: ocsf.connection_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `_duration_ms` to `ocsf.duration` + sources: + - _duration_ms + target: ocsf.duration + preserveSource: false + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `conn_state` to `ocsf.status_detail` + sources: + - conn_state + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@conn_state:(SF OR S1 OR S2 OR S3 OR OTH OR RSTO OR RSTRH OR SH OR SHR)" + name: Success + id: 1 + - filter: + query: "@conn_state:(S0 OR RSTOS0 OR RSTR OR REJ)" + name: Failure + id: 2 + - filter: + query: "*" + name: Other + id: 99 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: "99" + sources: + ocsf.status: + - conn_state + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `_total_bytes` to `ocsf.traffic.bytes` + sources: + - _total_bytes + target: ocsf.traffic.bytes + preserveSource: false + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `resp_bytes` to `ocsf.traffic.bytes_in` + sources: + - resp_bytes + target: ocsf.traffic.bytes_in + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `missed_bytes` to `ocsf.traffic.bytes_missed` + sources: + - missed_bytes + target: ocsf.traffic.bytes_missed + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `orig_bytes` to `ocsf.traffic.bytes_out` + sources: + - orig_bytes + target: ocsf.traffic.bytes_out + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `_total_packets` to `ocsf.traffic.packets` + sources: + - _total_packets + target: ocsf.traffic.packets + preserveSource: false + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `resp_pkts` to `ocsf.traffic.packets_in` + sources: + - resp_pkts + target: ocsf.traffic.packets_in + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `orig_pkts` to `ocsf.traffic.packets_out` + sources: + - orig_pkts + target: ocsf.traffic.packets_out + preserveSource: true + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: Network Activity + classUid: 4001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Network Activity [4001] - ssl + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(ssl OR ssl_red)" + processors: + - type: string-builder-processor + name: Set JA3 hash algorithm name + enabled: true + template: MD5 + target: ocsf.tls.ja3_hash.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set JA3 hash algorithm id + enabled: true + template: "1" + target: _ja3_algorithm_id + replaceMissing: false + - type: string-builder-processor + name: Set JA3S hash algorithm name + enabled: true + template: MD5 + target: ocsf.tls.ja3s_hash.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set JA3S hash algorithm id + enabled: true + template: "1" + target: _ja3s_algorithm_id + replaceMissing: false + - type: schema-processor + name: Apply OCSF schema for 4001 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@established:true" + name: Open + id: 1 + - filter: + query: "@established:false" + name: Fail + id: 4 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-remapper + name: Map `server_name` to `ocsf.dst_endpoint.hostname` + sources: + - server_name + target: ocsf.dst_endpoint.hostname + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ts` to `ocsf.metadata.original_time` + sources: + - ts + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@established:true" + name: Success + id: 1 + - filter: + query: "@established:false" + name: Failure + id: 2 + targets: + name: ocsf.status + id: ocsf.status_id + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `cipher` to `ocsf.tls.cipher` + sources: + - cipher + target: ocsf.tls.cipher + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.tls.ja3_hash.algorithm` to `ocsf.tls.ja3_hash.algorithm` + sources: + - ocsf.tls.ja3_hash.algorithm + target: ocsf.tls.ja3_hash.algorithm + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `_ja3_algorithm_id` to `ocsf.tls.ja3_hash.algorithm_id` + sources: + - _ja3_algorithm_id + target: ocsf.tls.ja3_hash.algorithm_id + preserveSource: false + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ja3` to `ocsf.tls.ja3_hash.value` + sources: + - ja3 + target: ocsf.tls.ja3_hash.value + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.tls.ja3s_hash.algorithm` to `ocsf.tls.ja3s_hash.algorithm` + sources: + - ocsf.tls.ja3s_hash.algorithm + target: ocsf.tls.ja3s_hash.algorithm + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `_ja3s_algorithm_id` to `ocsf.tls.ja3s_hash.algorithm_id` + sources: + - _ja3s_algorithm_id + target: ocsf.tls.ja3s_hash.algorithm_id + preserveSource: false + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ja3s` to `ocsf.tls.ja3s_hash.value` + sources: + - ja3s + target: ocsf.tls.ja3s_hash.value + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `server_name` to `ocsf.tls.sni` + sources: + - server_name + target: ocsf.tls.sni + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `version` to `ocsf.tls.version` + sources: + - version + target: ocsf.tls.version + preserveSource: true + overrideOnConflict: true + schema: + schemaType: ocsf + version: 1.5.0 + className: Network Activity + classUid: 4001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Network Activity [4001] - weird + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:weird_red" + processors: + - type: string-builder-processor + name: Lowercase source for protocol + enabled: true + template: "%{source}" + target: _protocol_name + replaceMissing: false + - type: schema-processor + name: Apply OCSF schema for 4001 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "*" + name: Traffic + id: 6 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.connection_info.direction_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.connection_info.direction + id: ocsf.connection_info.direction_id + - type: schema-remapper + name: Map `_protocol_name` to `ocsf.connection_info.protocol_name` + sources: + - _protocol_name + target: ocsf.connection_info.protocol_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.connection_info.uid` + sources: + - uid + target: ocsf.connection_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `name` to `ocsf.message` + sources: + - name + target: ocsf.message + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `name` to `ocsf.metadata.event_code` + sources: + - name + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `name` to `ocsf.status_detail` + sources: + - name + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: Network Activity + classUid: 4001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class HTTP Activity [4002] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(http OR http_red)" + processors: + - type: schema-processor + name: Apply OCSF schema for 4002 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@http.method:CONNECT" + name: Connect + id: 1 + - filter: + query: "@http.method:DELETE" + name: Delete + id: 2 + - filter: + query: "@http.method:GET" + name: Get + id: 3 + - filter: + query: "@http.method:HEAD" + name: Head + id: 4 + - filter: + query: "@http.method:OPTIONS" + name: Options + id: 5 + - filter: + query: "@http.method:POST" + name: Post + id: 6 + - filter: + query: "@http.method:PUT" + name: Put + id: 7 + - filter: + query: "@http.method:TRACE" + name: Trace + id: 8 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - http.method + - type: schema-category-mapper + name: ocsf.connection_info.direction_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.connection_info.direction + id: ocsf.connection_info.direction_id + - type: schema-remapper + name: Map `version` to `ocsf.connection_info.protocol_ver` + sources: + - version + target: ocsf.connection_info.protocol_ver + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.connection_info.uid` + sources: + - uid + target: ocsf.connection_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `request_body_len` to `ocsf.http_request.body_length` + sources: + - request_body_len + target: ocsf.http_request.body_length + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `method` to `ocsf.http_request.http_method` + sources: + - method + target: ocsf.http_request.http_method + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.http_request.uid` + sources: + - uid + target: ocsf.http_request.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `host` to `ocsf.http_request.url.hostname` + sources: + - host + target: ocsf.http_request.url.hostname + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uri` to `ocsf.http_request.url.path` + sources: + - uri + target: ocsf.http_request.url.path + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.http_request.url.port` + sources: + - id.resp_p + target: ocsf.http_request.url.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `http_url` to `ocsf.http_request.url.url_string` + sources: + - http_url + target: ocsf.http_request.url.url_string + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `user_agent` to `ocsf.http_request.user_agent` + sources: + - user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `version` to `ocsf.http_request.version` + sources: + - version + target: ocsf.http_request.version + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `response_body_len` to `ocsf.http_response.body_length` + sources: + - response_body_len + target: ocsf.http_response.body_length + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `status_code` to `ocsf.http_response.code` + sources: + - status_code + target: ocsf.http_response.code + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `status_msg` to `ocsf.http_response.message` + sources: + - status_msg + target: ocsf.http_response.message + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "@http.status_code:[500 TO 599]" + name: High + id: 4 + - filter: + query: "@http.status_code:[400 TO 499]" + name: Medium + id: 3 + - filter: + query: "@http.status_code:[300 TO 399]" + name: Low + id: 2 + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `status_msg` to `ocsf.status_detail` + sources: + - status_msg + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@http.status_code:[200 TO 299]" + name: Success + id: 1 + - filter: + query: "@http.status_code:[400 TO 599]" + name: Failure + id: 2 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: "99" + sources: + ocsf.status: + - http.status_code + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: HTTP Activity + classUid: 4002 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class DNS Activity [4003] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(dns OR dns_red)" + processors: + - type: schema-processor + name: Apply OCSF schema for 4003 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@dns.answer.name:*" + name: Response + id: 2 + - filter: + query: "-@dns.answer.name:*" + name: Query + id: 1 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.connection_info.direction_id + categories: + - filter: + query: "@dns.answer.name:*" + name: Inbound + id: 1 + - filter: + query: "-@dns.answer.name:*" + name: Outbound + id: 2 + targets: + name: ocsf.connection_info.direction + id: ocsf.connection_info.direction_id + - type: schema-remapper + name: Map `proto` to `ocsf.connection_info.protocol_name` + sources: + - proto + target: ocsf.connection_info.protocol_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.connection_info.uid` + sources: + - uid + target: ocsf.connection_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `query, rcode_name` to `ocsf.message` + sources: + - query + - rcode_name + target: ocsf.message + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `qclass_name` to `ocsf.query.class` + sources: + - qclass_name + target: ocsf.query.class + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `query` to `ocsf.query.hostname` + sources: + - query + target: ocsf.query.hostname + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `qtype_name` to `ocsf.query.type` + sources: + - qtype_name + target: ocsf.query.type + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `rcode_name` to `ocsf.rcode` + sources: + - rcode_name + target: ocsf.rcode + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.rcode_id + categories: + - filter: + query: "@rcode:0" + name: NoError + id: 0 + - filter: + query: "@rcode:1" + name: FormError + id: 1 + - filter: + query: "@rcode:2" + name: ServError + id: 2 + - filter: + query: "@rcode:3" + name: NXDomain + id: 3 + - filter: + query: "@rcode:4" + name: NotImp + id: 4 + - filter: + query: "@rcode:5" + name: Refused + id: 5 + - filter: + query: "@rcode:6" + name: YXDomain + id: 6 + - filter: + query: "@rcode:7" + name: YXRRSet + id: 7 + - filter: + query: "@rcode:8" + name: NXRRSet + id: 8 + - filter: + query: "@rcode:9" + name: NotAuth + id: 9 + - filter: + query: "@rcode:10" + name: NotZone + id: 10 + - filter: + query: "@rcode:11" + name: DSOTYPENI + id: 11 + - filter: + query: "@rcode:16" + name: BADSIG_VERS + id: 16 + - filter: + query: "@rcode:17" + name: BADKEY + id: 17 + - filter: + query: "@rcode:18" + name: BADTIME + id: 18 + - filter: + query: "@rcode:19" + name: BADMODE + id: 19 + - filter: + query: "@rcode:20" + name: BADNAME + id: 20 + - filter: + query: "@rcode:21" + name: BADALG + id: 21 + - filter: + query: "@rcode:22" + name: BADTRUNC + id: 22 + - filter: + query: "@rcode:23" + name: BADCOOKIE + id: 23 + targets: + name: ocsf.rcode + id: ocsf.rcode_id + fallback: + values: + ocsf.rcode: Other + ocsf.rcode_id: "99" + sources: + ocsf.rcode: + - rcode + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `rcode_name` to `ocsf.status_detail` + sources: + - rcode_name + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "(@rcode:0 OR @dns.flags.rcode:NOERROR)" + name: Success + id: 1 + - filter: + query: "(@rcode:3 OR @dns.flags.rcode:NXDOMAIN OR @rejected:true)" + name: Failure + id: 2 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: "99" + sources: + ocsf.status: + - dns.flags.rcode + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: DNS Activity + classUid: 4003 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class File Hosting Activity [6006] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:files" + processors: + - type: string-builder-processor + name: Stringify tx_hosts + enabled: true + template: "%{tx_hosts}" + target: _tx_hosts_str + replaceMissing: false + - type: string-builder-processor + name: Stringify rx_hosts + enabled: true + template: "%{rx_hosts}" + target: _rx_hosts_str + replaceMissing: false + - type: grok-parser + name: Extract first IP from tx_hosts + enabled: true + source: _tx_hosts_str + samples: + - '["10.104.10.60"]' + grok: + supportRules: "" + matchRules: 'g \[?"?%{ip:_tx_host}"?' + - type: grok-parser + name: Extract first IP from rx_hosts + enabled: true + source: _rx_hosts_str + samples: + - '["10.104.10.65"]' + grok: + supportRules: "" + matchRules: 'g \[?"?%{ip:_rx_host}"?' + - type: schema-processor + name: Apply OCSF schema for 6006 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@is_orig:true" + name: Upload + id: 1 + - filter: + query: "@is_orig:false" + name: Download + id: 2 + - filter: + query: "-@is_orig:*" + name: Unknown + id: 0 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-remapper + name: Map `uid` to `ocsf.actor.session.uid` + sources: + - uid + target: ocsf.actor.session.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `_rx_host` to `ocsf.dst_endpoint.ip` + sources: + - _rx_host + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.file.hashes` to `ocsf.file.hashes` + sources: + - ocsf.file.hashes + target: ocsf.file.hashes + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `mime_type` to `ocsf.file.mime_type` + sources: + - mime_type + target: ocsf.file.mime_type + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `fuid` to `ocsf.file.name` + sources: + - fuid + target: ocsf.file.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `seen_bytes, total_bytes` to `ocsf.file.size` + sources: + - seen_bytes + - total_bytes + target: ocsf.file.size + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-category-mapper + name: ocsf.file.type_id + categories: + - filter: + query: "*" + name: Regular File + id: 1 + targets: + name: ocsf.file.type_id + id: ocsf.file.type_id + - type: schema-remapper + name: Map `fuid` to `ocsf.file.uid` + sources: + - fuid + target: ocsf.file.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `_tx_host` to `ocsf.src_endpoint.ip` + sources: + - _tx_host + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@timedout:false" + name: Success + id: 1 + - filter: + query: "@timedout:true" + name: Failure + id: 2 + targets: + name: ocsf.status + id: ocsf.status_id + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + schema: + schemaType: ocsf + version: 1.5.0 + className: File Hosting Activity + classUid: 6006 + extensions: [] + profiles: [] diff --git a/zeek/assets/logs/zeek_tests.yaml b/zeek/assets/logs/zeek_tests.yaml index dc27f90be8adf..186c0bc8d548c 100644 --- a/zeek/assets/logs/zeek_tests.yaml +++ b/zeek/assets/logs/zeek_tests.yaml @@ -18,7 +18,7 @@ tests: message: "<134>Dec 26 01:35:11 machine-name {\"_path\":\"capture_loss\",\"_write_ts\":\"2023-12-12T05:52:50.756358Z\",\"ts\":\"2023-12-12T05:52:32.763303Z\",\"ts_delta\":15.235642194747925,\"peer\":\"zeek\",\"gaps\":3,\"acks\":316,\"percent_lost\":0.9493670886075949}" service: "capture_loss" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: |- @@ -73,6 +73,57 @@ tests: duration_sec: 3.0111899375915527 missed_bytes: 0 proto: "icmp" + ocsf: + severity: Informational + activity_name: Traffic + metadata: + uid: CcdWSj20NUmxZowq93 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 3 + ip: 185.64.148.0 + status_detail: OTH + duration: 3011 + status_id: 1 + connection_info: + boundary: Localhost + boundary_id: 1 + uid: CcdWSj20NUmxZowq93 + direction_id: 3 + protocol_name: icmp + direction: Lateral + class_uid: 4001 + activity_id: 6 + time: 1702360352763 + dst_endpoint: + port: 10 + ip: 185.64.148.0 + severity_id: 1 + class_name: Network Activity + traffic: + bytes_out: 234 + bytes_in: 0 + packets_out: 3 + bytes: 234 + packets_in: 0 + packets: 3 + bytes_missed: 0 + status: Success + orig_bytes: 234 + missed_bytes: 0 + duration: 3.0111899375915527 + resp_bytes: 0 + proto: icmp + id: + orig_p: 3 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 10 message: |- { "resp_pkts" : 0, @@ -100,7 +151,7 @@ tests: } service: "conn" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: |- @@ -153,17 +204,75 @@ tests: ip: "185.64.148.0" port: 80 orig_fuids: - - "FKhxp22QnZua8NWY2" + - "FKhxp22QnZua8NWY2" orig_mime_types: - - "application/ocsp-request" + - "application/ocsp-request" resp_fuids: - - "F9Zqr71YzMZPVZ4dbe" + - "F9Zqr71YzMZPVZ4dbe" resp_mime_types: - - "application/ocsp-response" + - "application/ocsp-response" status_msg: "OK" trans_depth: 1 ts: "2023-12-12T05:52:32.763303Z" uid: "CBQBvs1hob384mp2lb" + ocsf: + http_response: + code: 200 + message: OK + body_length: 1434 + severity: Informational + activity_name: Post + metadata: + uid: CBQBvs1hob384mp2lb + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 47210 + ip: 185.64.148.0 + status_detail: OK + status_id: 1 + connection_info: + uid: CBQBvs1hob384mp2lb + protocol_ver: '1.1' + direction_id: 0 + direction: Unknown + class_uid: 4002 + activity_id: 6 + http_request: + uid: CBQBvs1hob384mp2lb + http_method: POST + version: '1.1' + url: + path: /rootr3 + hostname: host.com + port: 80 + url_string: http://host.com/rootr3 + user_agent: LIBCURL + body_length: 83 + time: 1702360352763 + dst_endpoint: + port: 80 + ip: 185.64.148.0 + severity_id: 1 + class_name: HTTP Activity + status: Success + status_code: 200 + method: POST + request_body_len: 83 + uri: /rootr3 + version: '1.1' + host: host.com + id: + orig_p: 47210 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 80 + response_body_len: 1434 + user_agent: LIBCURL message: |- { "status_code" : 200, @@ -193,7 +302,7 @@ tests: } service: "http" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: |- @@ -229,14 +338,14 @@ tests: RD: false TC: false TTLs: - - 30.0 + - 30 Z: 0 _path: "dns" _write_ts: "2023-12-12T05:52:50.756358Z" dns: answer: name: - - "185.64.148.0" + - "185.64.148.0" flags: rcode: "NOERROR" id: 38706 @@ -272,6 +381,52 @@ tests: uid: "CsOSdHqRMu62rNs31" zeek: proto: "udp" + ocsf: + severity: Informational + activity_name: Response + metadata: + uid: CsOSdHqRMu62rNs31 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 58013 + ip: 185.64.148.0 + query: + hostname: win2k16-1-159 + rcode: NoError + message: win2k16-1-159 + status_detail: NOERROR + rcode_id: 0 + status_id: 1 + connection_info: + uid: CsOSdHqRMu62rNs31 + direction_id: 1 + protocol_name: udp + direction: Inbound + class_uid: 4003 + activity_id: 2 + time: 1702360352763 + dst_endpoint: + port: 5355 + ip: 185.64.148.0 + severity_id: 1 + class_name: DNS Activity + status: Success + query: win2k16-1-159 + answers: + - 185.64.148.0 + trans_id: 38706 + rcode_name: NOERROR + proto: udp + id: + orig_p: 58013 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 5355 message: |- { "AA" : false, @@ -300,7 +455,7 @@ tests: } service: "dns" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: |- @@ -324,7 +479,7 @@ tests: service: "reporter" status: "info" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: "<134>Jan 12 18:13:46 machine-name {\"_path\":\"datared\",\"_write_ts\":\"2024-01-13T00:13:42.817478Z\",\"ts\":\"2024-01-13T00:13:42.817478Z\",\"conn_red\":1310,\"conn_total\":18460,\"dns_red\":1141,\"dns_total\":1604,\"files_red\":313,\"files_total\":796,\"http_red\":140,\"http_total\":140,\"ssl_red\":215,\"ssl_total\":227,\"weird_red\":20,\"weird_total\":20}" @@ -350,52 +505,8 @@ tests: message: "<134>Jan 12 18:13:46 machine-name {\"_path\":\"datared\",\"_write_ts\":\"2024-01-13T00:13:42.817478Z\",\"ts\":\"2024-01-13T00:13:42.817478Z\",\"conn_red\":1310,\"conn_total\":18460,\"dns_red\":1141,\"dns_total\":1604,\"files_red\":313,\"files_total\":796,\"http_red\":140,\"http_total\":140,\"ssl_red\":215,\"ssl_total\":227,\"weird_red\":20,\"weird_total\":20}" service: "datared" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1705104822817 - - - sample: "<134>Mar 6 20:22:19 machine-name {\"_path\":\"files_red\",\"_write_ts\":\"2024-03-07T02:22:18.145479Z\",\"ts\":[\"2024-03-07T02:12:12.099810Z\"],\"fuid\":\"FJcjJV2mrRbxT0TWi\",\"tx_hosts\":[\"10.10.10.10\"],\"rx_hosts\":[\"20.20.20.20\"],\"conn_uids\":[\"CALvyD1AyopngTm5Xh\"],\"source\":\"HTTP\",\"depth\":0,\"analyzers\":[\"SHA1\",\"MD5\",\"DATA_EVENT\",\"SHA256\"],\"local_orig\":true,\"is_orig\":false,\"seen_bytes\":163,\"total_bytes\":163,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"extracted\":[],\"md5\":\"e335c05220a3858e858d2026071e3se2\",\"sha1\":\"8514de756fa993adb449c282affe84c752bab495\",\"sha256\":\"2733c2e83762f8a9542ebacf40642505fee0247295161ced264b44d2ad6f9456\",\"num\":1}" - service: "corelight" - result: - custom: - _path: "files_red" - _write_ts: "2024-03-07T02:22:18.145479Z" - analyzers: - - "SHA1" - - "MD5" - - "DATA_EVENT" - - "SHA256" - conn_uids: - - "CALvyD1AyopngTm5Xh" - depth: 0 - dest_host: - - "10.10.10.10" - fuid: "FJcjJV2mrRbxT0TWi" - is_orig: false - local_orig: true - md5: "e335c05220a3858e858d2026071e3se2" - num: 1 - orig_host: - - "20.20.20.20" - rx_hosts: - - "20.20.20.20" - sha1: "8514de756fa993adb449c282affe84c752bab495" - sha256: "2733c2e83762f8a9542ebacf40642505fee0247295161ced264b44d2ad6f9456" - source: "HTTP" - timedout: false - ts: - - "2024-03-07T02:12:12.099810Z" - tx_hosts: - - "10.10.10.10" - zeek: - missing_bytes: 0 - overflow_bytes: 0 - seen_bytes: 163 - total_bytes: 163 - message: "<134>Mar 6 20:22:19 machine-name {\"_path\":\"files_red\",\"_write_ts\":\"2024-03-07T02:22:18.145479Z\",\"ts\":[\"2024-03-07T02:12:12.099810Z\"],\"fuid\":\"FJcjJV2mrRbxT0TWi\",\"tx_hosts\":[\"10.10.10.10\"],\"rx_hosts\":[\"20.20.20.20\"],\"conn_uids\":[\"CALvyD1AyopngTm5Xh\"],\"source\":\"HTTP\",\"depth\":0,\"analyzers\":[\"SHA1\",\"MD5\",\"DATA_EVENT\",\"SHA256\"],\"local_orig\":true,\"is_orig\":false,\"seen_bytes\":163,\"total_bytes\":163,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"extracted\":[],\"md5\":\"e335c05220a3858e858d2026071e3se2\",\"sha1\":\"8514de756fa993adb449c282affe84c752bab495\",\"sha256\":\"2733c2e83762f8a9542ebacf40642505fee0247295161ced264b44d2ad6f9456\",\"num\":1}" - service: "files_red" - tags: - - "source:LOGS_SOURCE" - timestamp: 1709778138145 - sample: "<134>Mar 6 22:39:30 machine-name {\"_path\":\"dns_red\",\"_write_ts\":\"2024-03-07T04:39:28.580374Z\",\"ts\":\"2024-03-07T04:38:40.085451Z\",\"uid\":\"CyArTY2KEgcygwMLi1\",\"id.orig_h\":\"10.10.10.10\",\"id.orig_p\":123,\"id.resp_h\":\"20.20.20.20\",\"id.resp_p\":321,\"query\":\"time.missouri.edu\",\"qtype_name\":\"A\",\"num\":4}" service: "corelight" @@ -418,10 +529,47 @@ tests: num: 4 ts: "2024-03-07T04:38:40.085451Z" uid: "CyArTY2KEgcygwMLi1" + ocsf: + severity: Informational + activity_name: Query + metadata: + uid: CyArTY2KEgcygwMLi1 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 123 + ip: 10.10.10.10 + query: + hostname: time.missouri.edu + type: A + message: time.missouri.edu + connection_info: + uid: CyArTY2KEgcygwMLi1 + direction_id: 2 + direction: Outbound + class_uid: 4003 + activity_id: 1 + time: 1709786320085 + dst_endpoint: + port: 321 + ip: 20.20.20.20 + severity_id: 1 + class_name: DNS Activity + qtype_name: A + query: time.missouri.edu + id: + orig_p: 123 + resp_h: 20.20.20.20 + orig_h: 10.10.10.10 + resp_p: 321 message: "<134>Mar 6 22:39:30 machine-name {\"_path\":\"dns_red\",\"_write_ts\":\"2024-03-07T04:39:28.580374Z\",\"ts\":\"2024-03-07T04:38:40.085451Z\",\"uid\":\"CyArTY2KEgcygwMLi1\",\"id.orig_h\":\"10.10.10.10\",\"id.orig_p\":123,\"id.resp_h\":\"20.20.20.20\",\"id.resp_p\":321,\"query\":\"time.missouri.edu\",\"qtype_name\":\"A\",\"num\":4}" service: "dns_red" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1709786320085 - sample: "<134>Nov 19 22:36:37 machine-name {\"_path\":\"etc_viz\",\"_system_name\":\"machine-name\",\"_write_ts\":\"2024-03-28T00:01:04.742415Z\",\"c2s_viz.clr_frac\":0.0,\"c2s_viz.enc_dev\":0.30151134457776363,\"c2s_viz.enc_frac\":1.0,\"c2s_viz.pdu1_enc\":true,\"c2s_viz.size\":198,\"err\":null,\"s2c_viz.clr_frac\":0.0,\"s2c_viz.enc_dev\":0.6957566520492713,\"s2c_viz.enc_frac\":1.0,\"s2c_viz.pdu1_enc\":true,\"s2c_viz.size\":818,\"server_a\":\"10.10.10.10\",\"server_p\":443,\"service\":[],\"ts\":\"2024-03-28T00:00:52.656388Z\",\"uid\":\"CV79JD7onkfdf7WDa\",\"viz_stat\":\"Ee!!\"}" @@ -432,9 +580,9 @@ tests: _system_name: "machine-name" _write_ts: "2024-03-28T00:01:04.742415Z" c2s_viz: - clr_frac: 0.0 + clr_frac: 0 enc_dev: 0.30151134457776363 - enc_frac: 1.0 + enc_frac: 1 pdu1_enc: true size: 198 network: @@ -442,16 +590,853 @@ tests: ip: "10.10.10.10" port: 443 s2c_viz: - clr_frac: 0.0 + clr_frac: 0 enc_dev: 0.6957566520492713 - enc_frac: 1.0 + enc_frac: 1 pdu1_enc: true size: 818 ts: "2024-03-28T00:00:52.656388Z" uid: "CV79JD7onkfdf7WDa" viz_stat: "Ee!!" + server_p: 443 message: "<134>Nov 19 22:36:37 machine-name {\"_path\":\"etc_viz\",\"_system_name\":\"machine-name\",\"_write_ts\":\"2024-03-28T00:01:04.742415Z\",\"c2s_viz.clr_frac\":0.0,\"c2s_viz.enc_dev\":0.30151134457776363,\"c2s_viz.enc_frac\":1.0,\"c2s_viz.pdu1_enc\":true,\"c2s_viz.size\":198,\"err\":null,\"s2c_viz.clr_frac\":0.0,\"s2c_viz.enc_dev\":0.6957566520492713,\"s2c_viz.enc_frac\":1.0,\"s2c_viz.pdu1_enc\":true,\"s2c_viz.size\":818,\"server_a\":\"10.10.10.10\",\"server_p\":443,\"service\":[],\"ts\":\"2024-03-28T00:00:52.656388Z\",\"uid\":\"CV79JD7onkfdf7WDa\",\"viz_stat\":\"Ee!!\"}" service: "etc_viz" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1711584052656 + - + sample: "<134>May 11 18:47:07 test-system {\"_path\":\"conn\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T18:47:07.850764Z\",\"ts\":\"2026-05-11T18:47:02.848960Z\",\"uid\":\"12345678901234567890\",\"id.orig_h\":\"10.10.50.1\",\"id.orig_p\":60704,\"id.resp_h\":\"10.10.60.6\",\"id.resp_p\":88,\"proto\":\"tcp\",\"service\":\"krb_tcp\",\"orig_bytes\":227,\"resp_bytes\":195,\"conn_state\":\"RSTR\",\"local_orig\":true,\"local_resp\":true,\"missed_bytes\":0,\"history\":\"ShADTdFar\",\"orig_pkts\":5,\"resp_pkts\":7,\"orig_ip_bytes\":666,\"resp_ip_bytes\":499,\"community_id\":\"1:abc123\",\"tunnel_parents\":[\"test\",\"test2\"]}" + service: "corelight" + result: + custom: + resp_pkts: 7 + resp_ip_bytes: 499 + local_orig: true + _write_ts: '2026-05-11T18:47:07.850764Z' + orig_ip_bytes: 666 + zeek: + proto: tcp + missed_bytes: 0 + orig_pkts: 5 + _path: conn + history: ShADTdFar + tunnel_parents: + - test + - test2 + connection_state: Responder sent a RST + network: + bytes_written: 195 + destination: + port: 88 + ip: 10.10.60.6 + client: + port: 60704 + ip: 10.10.50.1 + bytes_read: 227 + local_resp: true + uid: '12345678901234567890' + community_id: 1:abc123 + ocsf: + severity: Informational + activity_name: Close + metadata: + uid: '12345678901234567890' + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 60704 + ip: 10.10.50.1 + status_detail: RSTR + status_id: 2 + connection_info: + boundary: Localhost + boundary_id: 1 + uid: '12345678901234567890' + direction_id: 3 + community_uid: 1:abc123 + flag_history: ShADTdFar + protocol_name: tcp + direction: Lateral + class_uid: 4001 + activity_id: 2 + time: 1778525222848 + dst_endpoint: + port: 88 + ip: 10.10.60.6 + severity_id: 1 + class_name: Network Activity + traffic: + bytes_out: 227 + bytes_in: 195 + packets_out: 5 + bytes: 422 + packets_in: 7 + packets: 12 + bytes_missed: 0 + status: Failure + _system_name: test-system + service: krb_tcp + conn_state: RSTR + ts: '2026-05-11T18:47:02.848960Z' + id: + orig_p: 60704 + resp_h: 10.10.60.6 + orig_h: 10.10.50.1 + resp_p: 88 + orig_bytes: 227 + missed_bytes: 0 + resp_bytes: 195 + proto: tcp + service: conn + message: <134>May 11 18:47:07 test-system {"_path":"conn","_system_name":"test-system","_write_ts":"2026-05-11T18:47:07.850764Z","ts":"2026-05-11T18:47:02.848960Z","uid":"12345678901234567890","id.orig_h":"10.10.50.1","id.orig_p":60704,"id.resp_h":"10.10.60.6","id.resp_p":88,"proto":"tcp","service":"krb_tcp","orig_bytes":227,"resp_bytes":195,"conn_state":"RSTR","local_orig":true,"local_resp":true,"missed_bytes":0,"history":"ShADTdFar","orig_pkts":5,"resp_pkts":7,"orig_ip_bytes":666,"resp_ip_bytes":499,"community_id":"1:abc123","tunnel_parents":["test","test2"]} + tags: + - source:LOGS_SOURCE + timestamp: 1778525222848 + - + sample: "<134>May 11 18:46:03 ndr-pvg3-1 {\"_path\":\"conn_long\",\"_system_name\":\"ndr-pvg3-1\",\"_write_ts\":\"2026-05-11T18:46:03.887017Z\",\"ts\":\"2026-05-11T18:36:03.886935Z\",\"uid\":\"12345678901234568\",\"id.orig_h\":\"10.250.5.75\",\"id.orig_p\":63231,\"id.resp_h\":\"172.64.5.1\",\"id.resp_p\":443,\"id.vlan\":1000,\"proto\":\"tcp\",\"service\":\"ssl\",\"duration\":600.0000820159912,\"orig_bytes\":7071,\"resp_bytes\":18980,\"conn_state\":\"S1\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"ShADadtT\",\"orig_pkts\":32,\"resp_pkts\":37,\"orig_ip_bytes\":8380,\"resp_ip_bytes\":20532,\"community_id\":\"1:def456\",\"corelight_shunted\":false}" + service: "corelight" + result: + custom: + resp_pkts: 37 + corelight_shunted: false + connection_state: Connection established - not terminated + network: + bytes_written: 18980 + destination: + port: 443 + ip: 172.64.5.1 + client: + port: 63231 + ip: 10.250.5.75 + bytes_read: 7071 + local_resp: false + uid: '12345678901234568' + community_id: 1:def456 + ocsf: + severity: Informational + activity_name: Traffic + metadata: + uid: '12345678901234568' + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 63231 + ip: 10.250.5.75 + status_detail: S1 + duration: 600000 + status_id: 1 + connection_info: + boundary: External + boundary_id: 3 + uid: '12345678901234568' + direction_id: 2 + community_uid: 1:def456 + flag_history: ShADadtT + protocol_name: tcp + direction: Outbound + class_uid: 4001 + activity_id: 6 + time: 1778524563886 + dst_endpoint: + port: 443 + ip: 172.64.5.1 + severity_id: 1 + class_name: Network Activity + traffic: + bytes_out: 7071 + bytes_in: 18980 + packets_out: 32 + bytes: 26051 + packets_in: 37 + packets: 69 + bytes_missed: 0 + status: Success + conn_state: S1 + id: + vlan: 1000 + orig_p: 63231 + resp_h: 172.64.5.1 + orig_h: 10.250.5.75 + resp_p: 443 + resp_ip_bytes: 20532 + local_orig: true + _write_ts: '2026-05-11T18:46:03.887017Z' + orig_ip_bytes: 8380 + zeek: + proto: tcp + missed_bytes: 0 + duration_sec: 600.0000820159912 + orig_pkts: 32 + _path: conn_long + history: ShADadtT + _system_name: ndr-pvg3-1 + service: ssl + ts: '2026-05-11T18:36:03.886935Z' + duration: 600.0000820159912 + orig_bytes: 7071 + missed_bytes: 0 + resp_bytes: 18980 + proto: tcp + service: conn_long + message: <134>May 11 18:46:03 ndr-pvg3-1 {"_path":"conn_long","_system_name":"ndr-pvg3-1","_write_ts":"2026-05-11T18:46:03.887017Z","ts":"2026-05-11T18:36:03.886935Z","uid":"12345678901234568","id.orig_h":"10.250.5.75","id.orig_p":63231,"id.resp_h":"172.64.5.1","id.resp_p":443,"id.vlan":1000,"proto":"tcp","service":"ssl","duration":600.0000820159912,"orig_bytes":7071,"resp_bytes":18980,"conn_state":"S1","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"ShADadtT","orig_pkts":32,"resp_pkts":37,"orig_ip_bytes":8380,"resp_ip_bytes":20532,"community_id":"1:def456","corelight_shunted":false} + tags: + - source:LOGS_SOURCE + timestamp: 1778524563886 + - + sample: "<134>Dec 12 05:52:50 machine-name {\"_path\":\"http\",\"_write_ts\":\"2023-12-12T05:52:50.756358Z\",\"ts\":\"2023-12-12T05:52:32.763303Z\",\"uid\":\"CBQBvs1hob384mp2lb\",\"id.orig_h\":\"185.64.148.0\",\"id.orig_p\":47210,\"id.resp_h\":\"185.64.148.0\",\"id.resp_p\":80,\"method\":\"POST\",\"host\":\"host.com\",\"uri\":\"/rootr3\",\"version\":\"1.1\",\"user_agent\":\"LIBCURL\",\"request_body_len\":83,\"response_body_len\":1434,\"status_code\":200,\"status_msg\":\"OK\",\"trans_depth\":1,\"orig_mime_types\":[\"application/ocsp-request\"],\"resp_mime_types\":[\"application/ocsp-response\"],\"orig_fuids\":[\"FKhxp22QnZua8NWY2\"],\"resp_fuids\":[\"F9Zqr71YzMZPVZ4dbe\"]}" + service: "corelight" + result: + custom: + _write_ts: '2023-12-12T05:52:50.756358Z' + _path: http + orig_mime_types: + - application/ocsp-request + http_url: http://host.com/rootr3 + network: + bytes_written: 1434 + destination: + port: 80 + ip: 185.64.148.0 + client: + port: 47210 + ip: 185.64.148.0 + bytes_read: 83 + uid: CBQBvs1hob384mp2lb + resp_mime_types: + - application/ocsp-response + trans_depth: 1 + orig_fuids: + - FKhxp22QnZua8NWY2 + ocsf: + http_response: + code: 200 + message: OK + body_length: 1434 + severity: Informational + activity_name: Post + metadata: + uid: CBQBvs1hob384mp2lb + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 47210 + ip: 185.64.148.0 + status_detail: OK + status_id: 1 + connection_info: + uid: CBQBvs1hob384mp2lb + protocol_ver: '1.1' + direction_id: 0 + direction: Unknown + class_uid: 4002 + activity_id: 6 + http_request: + uid: CBQBvs1hob384mp2lb + http_method: POST + version: '1.1' + url: + path: /rootr3 + hostname: host.com + port: 80 + url_string: http://host.com/rootr3 + user_agent: LIBCURL + body_length: 83 + time: 1702360352763 + dst_endpoint: + port: 80 + ip: 185.64.148.0 + severity_id: 1 + class_name: HTTP Activity + status: Success + status_msg: OK + http: + url_details: + path: /rootr3 + host: host.com + status_code: 200 + method: POST + useragent: LIBCURL + version: '1.1' + ts: '2023-12-12T05:52:32.763303Z' + resp_fuids: + - F9Zqr71YzMZPVZ4dbe + status_code: 200 + method: POST + request_body_len: 83 + uri: /rootr3 + version: '1.1' + host: host.com + id: + orig_p: 47210 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 80 + response_body_len: 1434 + user_agent: LIBCURL + service: http + message: <134>Dec 12 05:52:50 machine-name {"_path":"http","_write_ts":"2023-12-12T05:52:50.756358Z","ts":"2023-12-12T05:52:32.763303Z","uid":"CBQBvs1hob384mp2lb","id.orig_h":"185.64.148.0","id.orig_p":47210,"id.resp_h":"185.64.148.0","id.resp_p":80,"method":"POST","host":"host.com","uri":"/rootr3","version":"1.1","user_agent":"LIBCURL","request_body_len":83,"response_body_len":1434,"status_code":200,"status_msg":"OK","trans_depth":1,"orig_mime_types":["application/ocsp-request"],"resp_mime_types":["application/ocsp-response"],"orig_fuids":["FKhxp22QnZua8NWY2"],"resp_fuids":["F9Zqr71YzMZPVZ4dbe"]} + tags: + - source:LOGS_SOURCE + timestamp: 1702360352763 + - + sample: "<134>Dec 12 05:52:50 machine-name {\"_path\":\"dns\",\"_write_ts\":\"2023-12-12T05:52:50.756358Z\",\"ts\":\"2023-12-12T05:52:32.763303Z\",\"uid\":\"CsOSdHqRMu62rNs31\",\"id.orig_h\":\"185.64.148.0\",\"id.orig_p\":58013,\"id.resp_h\":\"185.64.148.0\",\"id.resp_p\":5355,\"proto\":\"udp\",\"trans_id\":38706,\"rcode\":0,\"rcode_name\":\"NOERROR\",\"query\":\"win2k16-1-159\",\"answers\":[\"185.64.148.0\"],\"TTLs\":[30.0],\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"rejected\":false}" + service: "corelight" + result: + custom: + TTLs: + - 30 + AA: false + rejected: false + _write_ts: '2023-12-12T05:52:50.756358Z' + dns: + question: + name: win2k16-1-159 + answer: + name: + - 185.64.148.0 + flags: + rcode: NOERROR + id: 38706 + zeek: + proto: udp + rcode: 0 + _path: dns + TC: false + RA: false + network: + destination: + port: 5355 + ip: 185.64.148.0 + client: + geoip: + continent: + code: EU + name: Europe + country: + name: France + iso_code: FR + subdivision: + name: Île-de-France + iso_code: FR-IDF + city: + name: Paris + timezone: Europe/Paris + ipAddress: 185.64.148.0 + location: + latitude: 48.90654 + longitude: 2.33339 + port: 58013 + ip: 185.64.148.0 + uid: CsOSdHqRMu62rNs31 + RD: false + ocsf: + severity: Informational + activity_name: Response + metadata: + uid: CsOSdHqRMu62rNs31 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 58013 + ip: 185.64.148.0 + query: + hostname: win2k16-1-159 + rcode: NoError + message: win2k16-1-159 + status_detail: NOERROR + rcode_id: 0 + status_id: 1 + connection_info: + uid: CsOSdHqRMu62rNs31 + direction_id: 1 + protocol_name: udp + direction: Inbound + class_uid: 4003 + activity_id: 2 + time: 1702360352763 + dst_endpoint: + port: 5355 + ip: 185.64.148.0 + severity_id: 1 + class_name: DNS Activity + status: Success + Z: 0 + ts: '2023-12-12T05:52:32.763303Z' + query: win2k16-1-159 + answers: + - 185.64.148.0 + trans_id: 38706 + rcode_name: NOERROR + proto: udp + id: + orig_p: 58013 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 5355 + service: dns + message: <134>Dec 12 05:52:50 machine-name {"_path":"dns","_write_ts":"2023-12-12T05:52:50.756358Z","ts":"2023-12-12T05:52:32.763303Z","uid":"CsOSdHqRMu62rNs31","id.orig_h":"185.64.148.0","id.orig_p":58013,"id.resp_h":"185.64.148.0","id.resp_p":5355,"proto":"udp","trans_id":38706,"rcode":0,"rcode_name":"NOERROR","query":"win2k16-1-159","answers":["185.64.148.0"],"TTLs":[30.0],"AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} + tags: + - source:LOGS_SOURCE + timestamp: 1702360352763 + - + sample: "<134>May 11 17:22:02 test-system {\"_path\":\"ssl\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T17:22:02.759122Z\",\"ts\":\"2026-05-11T17:22:02.754532Z\",\"uid\":\"Cxu3mb4NZI2tAscrhb\",\"id.orig_h\":\"10.205.140.1\",\"id.orig_p\":43108,\"id.resp_h\":\"10.205.180.10\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\",\"curve\":\"secp256r1\",\"server_name\":null,\"resumed\":false,\"established\":true,\"ssl_history\":\"CsxknGIti\",\"ja3\":\"c34a54599a1fbaf1786bb6d633545a60\",\"ja3s\":\"03788d8896c247631984a250db971b74\",\"subject\":\"CN=*.org.test.io,O=Test Org,L=Los Angeles,ST=California,C=US\",\"issuer\":\"CN=Test Issuer,OU=Test OU,O=Test Org,C=US\",\"validation_status\":\"ok\",\"cert_chain_fps\":[\"3f5332c4631559ebf68bacbdf27d408g5d904bf5fa8e4f455c5274aae330ca96\",\"8bb2f6883fed289a521ba27c478482950874e167caccec6fc025990c0c46813f\"],\"port\":23367}" + service: "corelight" + result: + custom: + curve: secp256r1 + subject: CN=*.org.test.io,O=Test Org,L=Los Angeles,ST=California,C=US + ssl_history: CsxknGIti + issuer: CN=Test Issuer,OU=Test OU,O=Test Org,C=US + network: + destination: + port: 443 + ip: 10.205.180.10 + client: + port: 43108 + ip: 10.205.140.1 + uid: Cxu3mb4NZI2tAscrhb + ocsf: + severity: Informational + activity_name: Open + metadata: + uid: Cxu3mb4NZI2tAscrhb + product: + name: Zeek + vendor_name: Corelight + original_time: '2026-05-11T17:22:02.754532Z' + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 43108 + ip: 10.205.140.1 + status_id: 1 + class_uid: 4001 + activity_id: 1 + tls: + cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + ja3s_hash: + value: 03788d8896c247631984a250db971b74 + algorithm_id: 1 + algorithm: MD5 + ja3_hash: + value: c34a54599a1fbaf1786bb6d633545a60 + algorithm_id: 1 + algorithm: MD5 + version: TLSv12 + time: 1778520122754 + dst_endpoint: + port: 443 + ip: 10.205.180.10 + severity_id: 1 + class_name: Network Activity + status: Success + cert_chain_fps: + - 3f5332c4631559ebf68bacbdf27d408g5d904bf5fa8e4f455c5274aae330ca96 + - 8bb2f6883fed289a521ba27c478482950874e167caccec6fc025990c0c46813f + ja3: c34a54599a1fbaf1786bb6d633545a60 + cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + established: true + ja3s: 03788d8896c247631984a250db971b74 + _write_ts: '2026-05-11T17:22:02.759122Z' + _path: ssl + version: TLSv12 + _system_name: test-system + port: 23367 + validation_status: ok + resumed: false + ts: '2026-05-11T17:22:02.754532Z' + id: + orig_p: 43108 + resp_h: 10.205.180.10 + orig_h: 10.205.140.1 + resp_p: 443 + service: ssl + message: <134>May 11 17:22:02 test-system {"_path":"ssl","_system_name":"test-system","_write_ts":"2026-05-11T17:22:02.759122Z","ts":"2026-05-11T17:22:02.754532Z","uid":"Cxu3mb4NZI2tAscrhb","id.orig_h":"10.205.140.1","id.orig_p":43108,"id.resp_h":"10.205.180.10","id.resp_p":443,"version":"TLSv12","cipher":"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","curve":"secp256r1","server_name":null,"resumed":false,"established":true,"ssl_history":"CsxknGIti","ja3":"c34a54599a1fbaf1786bb6d633545a60","ja3s":"03788d8896c247631984a250db971b74","subject":"CN=*.org.test.io,O=Test Org,L=Los Angeles,ST=California,C=US","issuer":"CN=Test Issuer,OU=Test OU,O=Test Org,C=US","validation_status":"ok","cert_chain_fps":["3f5332c4631559ebf68bacbdf27d408g5d904bf5fa8e4f455c5274aae330ca96","8bb2f6883fed289a521ba27c478482950874e167caccec6fc025990c0c46813f"],"port":23367} + tags: + - source:LOGS_SOURCE + timestamp: 1778520122754 + - + sample: "<134>May 11 17:22:02 test-system {\"_path\":\"ssl_red\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T17:22:02.032606Z\",\"ts\":\"2026-05-11T17:22:02.023967Z\",\"uid\":\"CfSalo2IDfDMZeOJ02\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":54745,\"id.resp_h\":\"18.164.5.1\",\"id.resp_p\":443,\"id.vlan\":1000,\"version\":\"TLSv13\",\"cipher\":\"TLS_AES_128_GCM_SHA256\",\"curve\":\"x25519\",\"server_name\":\"public.test.app\",\"resumed\":false,\"established\":true,\"ssl_history\":\"CsiI\",\"ja3\":\"f4febc55ea12b31ae17cfb7e614afea4\",\"ja3s\":\"f4febc55ea12b31ae17cfb7e614afea4\",\"port\":64884}" + service: "corelight" + result: + custom: + cipher: TLS_AES_128_GCM_SHA256 + established: true + server_name: public.test.app + curve: x25519 + ja3s: f4febc55ea12b31ae17cfb7e614afea4 + _write_ts: '2026-05-11T17:22:02.032606Z' + ssl_history: CsiI + _path: ssl_red + version: TLSv13 + network: + destination: + port: 443 + ip: 18.164.5.1 + client: + port: 54745 + ip: 192.168.1.1 + uid: CfSalo2IDfDMZeOJ02 + ocsf: + severity: Informational + activity_name: Open + metadata: + uid: CfSalo2IDfDMZeOJ02 + product: + name: Zeek + vendor_name: Corelight + original_time: '2026-05-11T17:22:02.023967Z' + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 54745 + ip: 192.168.1.1 + status_id: 1 + class_uid: 4001 + activity_id: 1 + tls: + cipher: TLS_AES_128_GCM_SHA256 + ja3s_hash: + value: f4febc55ea12b31ae17cfb7e614afea4 + algorithm_id: 1 + algorithm: MD5 + ja3_hash: + value: f4febc55ea12b31ae17cfb7e614afea4 + algorithm_id: 1 + algorithm: MD5 + version: TLSv13 + sni: public.test.app + time: 1778520122023 + dst_endpoint: + hostname: public.test.app + port: 443 + ip: 18.164.5.1 + severity_id: 1 + class_name: Network Activity + status: Success + _system_name: test-system + port: 64884 + id: + vlan: 1000 + orig_p: 54745 + resp_h: 18.164.5.1 + orig_h: 192.168.1.1 + resp_p: 443 + resumed: false + ja3: f4febc55ea12b31ae17cfb7e614afea4 + ts: '2026-05-11T17:22:02.023967Z' + service: ssl_red + message: <134>May 11 17:22:02 test-system {"_path":"ssl_red","_system_name":"test-system","_write_ts":"2026-05-11T17:22:02.032606Z","ts":"2026-05-11T17:22:02.023967Z","uid":"CfSalo2IDfDMZeOJ02","id.orig_h":"192.168.1.1","id.orig_p":54745,"id.resp_h":"18.164.5.1","id.resp_p":443,"id.vlan":1000,"version":"TLSv13","cipher":"TLS_AES_128_GCM_SHA256","curve":"x25519","server_name":"public.test.app","resumed":false,"established":true,"ssl_history":"CsiI","ja3":"f4febc55ea12b31ae17cfb7e614afea4","ja3s":"f4febc55ea12b31ae17cfb7e614afea4","port":64884} + tags: + - source:LOGS_SOURCE + timestamp: 1778520122023 + - + sample: "<134>May 11 16:11:53 test-system {\"_path\":\"weird_red\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T16:11:53.944496Z\",\"ts\":\"2026-05-11T16:11:53.944496Z\",\"uid\":\"CYThCC2DZNVcGgpiI2\",\"id.orig_h\":\"10.250.182.4\",\"id.orig_p\":58438,\"id.resp_h\":\"180.153.200.12\",\"id.resp_p\":53,\"id.vlan\":1600,\"name\":\"DNS_truncated_quest_too_short\",\"source\":\"DNS\",\"notice\":false,\"peer\":\"worker-04\"}" + service: "corelight" + result: + custom: + _write_ts: '2026-05-11T16:11:53.944496Z' + _path: weird_red + source: DNS + network: + destination: + port: 53 + ip: 180.153.200.12 + client: + port: 58438 + ip: 10.250.182.4 + uid: CYThCC2DZNVcGgpiI2 + ocsf: + severity: Informational + activity_name: Traffic + metadata: + uid: CYThCC2DZNVcGgpiI2 + product: + name: Zeek + vendor_name: Corelight + event_code: DNS_truncated_quest_too_short + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 58438 + ip: 10.250.182.4 + message: DNS_truncated_quest_too_short + status_detail: DNS_truncated_quest_too_short + connection_info: + uid: CYThCC2DZNVcGgpiI2 + direction_id: 0 + protocol_name: DNS + direction: Unknown + class_uid: 4001 + activity_id: 6 + time: 1778515913944 + dst_endpoint: + port: 53 + ip: 180.153.200.12 + severity_id: 1 + class_name: Network Activity + _system_name: test-system + peer: worker-04 + name: DNS_truncated_quest_too_short + id: + vlan: 1600 + orig_p: 58438 + resp_h: 180.153.200.12 + orig_h: 10.250.182.4 + resp_p: 53 + ts: '2026-05-11T16:11:53.944496Z' + notice: false + _protocol_name: DNS + service: weird_red + message: <134>May 11 16:11:53 test-system {"_path":"weird_red","_system_name":"test-system","_write_ts":"2026-05-11T16:11:53.944496Z","ts":"2026-05-11T16:11:53.944496Z","uid":"CYThCC2DZNVcGgpiI2","id.orig_h":"10.250.182.4","id.orig_p":58438,"id.resp_h":"180.153.200.12","id.resp_p":53,"id.vlan":1600,"name":"DNS_truncated_quest_too_short","source":"DNS","notice":false,"peer":"worker-04"} + tags: + - source:LOGS_SOURCE + timestamp: 1778515913944 + - + sample: "<134>May 11 19:11:03 test-system {\"_path\":\"notice\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T19:11:03.521194Z\",\"ts\":\"2026-05-11T19:11:03.521194Z\",\"note\":\"ATTACK::Discovery\",\"msg\":\"Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins\",\"src\":\"10.10.10.15\",\"peer_descr\":\"manager\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600, \"uid\":\"12345678901234567890\", \"severity\":{\"name\":\"High\",\"id\":4}}" + service: "corelight" + result: + custom: + msg: Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins + suppress_for: 3600 + src: 10.10.10.15 + _write_ts: '2026-05-11T19:11:03.521194Z' + zeek: + note: ATTACK::Discovery + _path: notice + _is_alert_str: 'true' + peer_descr: manager + ocsf: + activity_name: Create + metadata: + product: + name: Zeek + vendor_name: Corelight + event_code: ATTACK::Discovery + version: 1.5.0 + uid: '12345678901234567890' + category_uid: 2 + category_name: Findings + is_alert: true + finding_info: + uid: ATTACK::Discovery + title: ATTACK::Discovery + status_id: 1 + class_uid: 2004 + activity_id: 1 + time: 1778526663521 + class_name: Detection Finding + status: New + severity: High + severity_id: 99 + _system_name: test-system + actions: + - Notice::ACTION_LOG + ts: '2026-05-11T19:11:03.521194Z' + note: ATTACK::Discovery + uid: '12345678901234567890' + severity: + name: High + id: 4 + service: notice + message: <134>May 11 19:11:03 test-system {"_path":"notice","_system_name":"test-system","_write_ts":"2026-05-11T19:11:03.521194Z","ts":"2026-05-11T19:11:03.521194Z","note":"ATTACK::Discovery","msg":"Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins","src":"10.10.10.15","peer_descr":"manager","actions":["Notice::ACTION_LOG"],"suppress_for":3600, "uid":"12345678901234567890", "severity":{"name":"High","id":4}} + tags: + - source:LOGS_SOURCE + timestamp: 1778526663521 + status: info + - + sample: "<134>May 11 17:59:59 test-system {\"_path\":\"suricata_corelight\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T17:59:59.359532Z\",\"ts\":\"2026-05-11T17:59:59.358114Z\",\"uid\":\"CN4bklsrGiWMsQhg3\",\"id.orig_h\":\"10.215.3.8\",\"id.orig_p\":65196,\"id.resp_h\":\"10.69.5.8\",\"id.resp_p\":80,\"id.vlan\":1123,\"proto\":\"tcp\",\"service\":\"http\",\"suri_id\":\"SOHaIDWJ5dBe\",\"flow_id\":2093130069465879,\"tx_id\":0,\"community_id\":\"1:Y4yC2kKf1Cm0QUkA/aBLSDxepcY=\",\"severity\":1,\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":2006380,\"rev\":17,\"signature\":\"ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted\",\"category\":\"Potential Corporate Privacy Violation\",\"severity\":1,\"metadata\":[\"confidence:Medium\",\"created_at:2010_07_30\",\"performance_impact:Significant\",\"signature_severity:Informational\",\"updated_at:2024_08_07\"]}}" + service: "corelight" + result: + custom: + alert_severity: High + _write_ts: '2026-05-11T17:59:59.359532Z' + suri_id: SOHaIDWJ5dBe + _path: suricata_corelight + _is_alert_str: 'true' + tx_id: 0 + network: + destination: + port: 80 + ip: 10.69.5.8 + client: + port: 65196 + ip: 10.215.3.8 + uid: CN4bklsrGiWMsQhg3 + community_id: 1:Y4yC2kKf1Cm0QUkA/aBLSDxepcY= + ocsf: + severity: High + activity_name: Create + metadata: + uid: CN4bklsrGiWMsQhg3 + product: + name: Suricata + vendor_name: Corelight + event_code: '2006380' + correlation_uid: 1:Y4yC2kKf1Cm0QUkA/aBLSDxepcY= + log_provider: Corelight + version: 1.5.0 + category_uid: 2 + category_name: Findings + confidence: Medium + is_alert: true + status_detail: allowed + finding_info: + uid_alt: SOHaIDWJ5dBe + title: ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted + analytic: + uid: '2006380' + type_id: 1 + name: ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted + type: Rule + uid: CN4bklsrGiWMsQhg3 + class_uid: 2004 + activity_id: 1 + time: 1778522399358 + severity_id: 4 + evidences: + - src_endpoint: + port: 65196 + ip: 10.215.3.8 + dst_endpoint: + port: 80 + ip: 10.69.5.8 + class_name: Detection Finding + confidence_id: 2 + _system_name: test-system + alert: + severity: 1 + signature_id: 2006380 + rev: 17 + metadata: + - confidence:Medium + - created_at:2010_07_30 + - performance_impact:Significant + - signature_severity:Informational + - updated_at:2024_08_07 + gid: 1 + signature: ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted + action: allowed + category: Potential Corporate Privacy Violation + service: http + flow_id: 2093130069465879 + proto: tcp + id: + vlan: 1123 + orig_p: 65196 + resp_h: 10.69.5.8 + orig_h: 10.215.3.8 + resp_p: 80 + ts: '2026-05-11T17:59:59.358114Z' + severity: 1 + service: suricata_corelight + message: <134>May 11 17:59:59 test-system {"_path":"suricata_corelight","_system_name":"test-system","_write_ts":"2026-05-11T17:59:59.359532Z","ts":"2026-05-11T17:59:59.358114Z","uid":"CN4bklsrGiWMsQhg3","id.orig_h":"10.215.3.8","id.orig_p":65196,"id.resp_h":"10.69.5.8","id.resp_p":80,"id.vlan":1123,"proto":"tcp","service":"http","suri_id":"SOHaIDWJ5dBe","flow_id":2093130069465879,"tx_id":0,"community_id":"1:Y4yC2kKf1Cm0QUkA/aBLSDxepcY=","severity":1,"alert":{"action":"allowed","gid":1,"signature_id":2006380,"rev":17,"signature":"ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted","category":"Potential Corporate Privacy Violation","severity":1,"metadata":["confidence:Medium","created_at:2010_07_30","performance_impact:Significant","signature_severity:Informational","updated_at:2024_08_07"]}} + tags: + - source:LOGS_SOURCE + timestamp: 1778522399358 + status: alert + - + sample: "<134>May 11 19:26:26 ndr-dub-stryker-DC-1 {\"_path\":\"files\",\"_system_name\":\"ndr-dub-stryker-DC-1\",\"_write_ts\":\"2026-05-11T19:26:26.082433Z\",\"ts\":\"2026-05-11T19:26:25.875206Z\",\"uid\":\"CjTuQU17IDvaVa8Nq2\",\"fuid\":\"FOPDsn3PdkiZsljcj2\",\"tx_hosts\":[\"10.104.10.60\"],\"rx_hosts\":[\"10.104.10.65\"],\"conn_uids\":[\"CjTuQU17IDvaGb8Nq2\"],\"source\":\"HTTP\",\"depth\":0,\"analyzers\":[\"SHA1\",\"MD5\",\"SHA256\",\"DATA_EVENT\"],\"local_orig\":true,\"is_orig\":false,\"seen_bytes\":253109,\"total_bytes\":253109,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"duration\":0.2072269916534424,\"mime_type\":\"text/json\",\"md5\":\"6e6ae0ed19f595687684faafae5499e13\",\"sha1\":\"f6578daa6d398c91398888b91a96d4c0e099c79c\",\"sha256\":\"a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f\",\"id.vlan\":1010}" + service: "corelight" + result: + custom: + timedout: false + sha256: a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f + source: HTTP + duration: 0.2072269916534424 + uid: CjTuQU17IDvaVa8Nq2 + analyzers: + - SHA1 + - MD5 + - SHA256 + - DATA_EVENT + ocsf: + severity: Informational + activity_name: Download + metadata: + uid: CjTuQU17IDvaVa8Nq2 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 6 + category_name: Application Activity + src_endpoint: + ip: 10.104.10.60 + actor: + session: + uid: CjTuQU17IDvaVa8Nq2 + file: + uid: FOPDsn3PdkiZsljcj2 + size: 253109 + mime_type: text/json + type_id: 1 + name: FOPDsn3PdkiZsljcj2 + status_id: 1 + class_uid: 6006 + activity_id: 2 + time: 1778527585875 + dst_endpoint: + ip: 10.104.10.65 + severity_id: 1 + class_name: File Hosting Activity + status: Success + fuid: FOPDsn3PdkiZsljcj2 + id: + vlan: 1010 + local_orig: true + _write_ts: '2026-05-11T19:26:26.082433Z' + rx_hosts: + - 10.104.10.65 + zeek: + total_bytes: 253109 + seen_bytes: 253109 + missing_bytes: 0 + overflow_bytes: 0 + orig_host: + - 10.104.10.65 + _path: files + is_orig: false + tx_hosts: + - 10.104.10.60 + sha1: f6578daa6d398c91398888b91a96d4c0e099c79c + depth: 0 + _system_name: ndr-dub-stryker-DC-1 + mime_type: text/json + dest_host: + - 10.104.10.60 + conn_uids: + - CjTuQU17IDvaGb8Nq2 + ts: '2026-05-11T19:26:25.875206Z' + md5: 6e6ae0ed19f595687684faafae5499e13 + _tx_hosts_str: 10.104.10.60 + _rx_hosts_str: 10.104.10.65 + _rx_host: 10.104.10.65 + _tx_host: 10.104.10.60 + total_bytes: 253109 + seen_bytes: 253109 + missing_bytes: 0 + service: files + message: <134>May 11 19:26:26 ndr-dub-stryker-DC-1 {"_path":"files","_system_name":"ndr-dub-stryker-DC-1","_write_ts":"2026-05-11T19:26:26.082433Z","ts":"2026-05-11T19:26:25.875206Z","uid":"CjTuQU17IDvaVa8Nq2","fuid":"FOPDsn3PdkiZsljcj2","tx_hosts":["10.104.10.60"],"rx_hosts":["10.104.10.65"],"conn_uids":["CjTuQU17IDvaGb8Nq2"],"source":"HTTP","depth":0,"analyzers":["SHA1","MD5","SHA256","DATA_EVENT"],"local_orig":true,"is_orig":false,"seen_bytes":253109,"total_bytes":253109,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"duration":0.2072269916534424,"mime_type":"text/json","md5":"6e6ae0ed19f595687684faafae5499e13","sha1":"f6578daa6d398c91398888b91a96d4c0e099c79c","sha256":"a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f","id.vlan":1010} + tags: + - source:LOGS_SOURCE + timestamp: 1778527585875 \ No newline at end of file From dec38cf426b6757e7cb8fa15202e1f28b46085d0 Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Thu, 14 May 2026 17:26:02 -0500 Subject: [PATCH 02/10] Fix validate-logs errors in zeek.yaml Resolve 36 validation errors flagged by the datadog-assets validator: - Add missing `overrideOnConflict: false` to 3 attribute-remappers - Fix 2 schema-remapper names to backtick individual fields - Rename 25 facets to match validator's canonical names and add `type: integer`/`facetType: range` where required - Remove 6 facets with unresolvable path conflicts (validator demanded unique paths with no canonical definition available) Co-Authored-By: Claude Opus 4.7 (1M context) --- zeek/assets/logs/zeek.yaml | 100 ++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 58 deletions(-) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index 0236b4e372be0..8a661babfcde6 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -402,22 +402,22 @@ facets: source: log - groups: - OCSF - name: Category UID + name: Category ID path: ocsf.category_uid source: log - groups: - OCSF - name: Category Name + name: Category path: ocsf.category_name source: log - groups: - OCSF - name: Class UID + name: Class ID path: ocsf.class_uid source: log - groups: - OCSF - name: Class Name + name: Class path: ocsf.class_name source: log - groups: @@ -442,12 +442,12 @@ facets: source: log - groups: - OCSF - name: Source Endpoint IP + name: Source IP Address path: ocsf.src_endpoint.ip source: log - groups: - OCSF - name: Destination Endpoint IP + name: Destination IP Address path: ocsf.dst_endpoint.ip source: log - groups: @@ -462,17 +462,17 @@ facets: source: log - groups: - OCSF - name: Product Vendor Name + name: Vendor Name path: ocsf.metadata.product.vendor_name source: log - groups: - OCSF - name: Finding Title + name: Finding Info Title path: ocsf.finding_info.title source: log - groups: - OCSF - name: Finding UID + name: Finding Info Unique ID path: ocsf.finding_info.uid source: log - groups: @@ -482,7 +482,7 @@ facets: source: log - groups: - OCSF - name: URL String + name: Request URL String path: ocsf.http_request.url.url_string source: log - groups: @@ -507,7 +507,7 @@ facets: source: log - groups: - OCSF - name: Actor Session UID + name: Session Unique ID path: ocsf.actor.session.uid source: log - groups: @@ -550,11 +550,6 @@ facets: name: Connection Info Flag History path: ocsf.connection_info.flag_history source: log - - groups: - - OCSF - name: Connection Info Protocol Name - path: ocsf.connection_info.protocol_name - source: log - groups: - OCSF name: Connection Info Protocol Ver @@ -575,16 +570,20 @@ facets: name: Dst Endpoint Hostname path: ocsf.dst_endpoint.hostname source: log - - groups: + - facetType: range + groups: - OCSF name: Dst Endpoint Port path: ocsf.dst_endpoint.port source: log - - groups: + type: integer + - facetType: range + groups: - OCSF - name: Duration + name: Duration Milliseconds path: ocsf.duration source: log + type: integer - groups: - OCSF name: Evidences @@ -605,9 +604,10 @@ facets: name: File Type ID path: ocsf.file.type_id source: log + type: integer - groups: - OCSF - name: File UID + name: File Unique ID path: ocsf.file.uid source: log - groups: @@ -642,17 +642,12 @@ facets: source: log - groups: - OCSF - name: HTTP Request UID - path: ocsf.http_request.uid - source: log - - groups: - - OCSF - name: HTTP Request URL Hostname + name: Url Hostname path: ocsf.http_request.url.hostname source: log - groups: - OCSF - name: HTTP Request URL Path + name: Url Path path: ocsf.http_request.url.path source: log - groups: @@ -662,14 +657,9 @@ facets: source: log - groups: - OCSF - name: HTTP Request User Agent + name: HTTP User-Agent path: ocsf.http_request.user_agent source: log - - groups: - - OCSF - name: HTTP Request Version - path: ocsf.http_request.version - source: log - groups: - OCSF name: HTTP Response Body Length @@ -677,34 +667,20 @@ facets: source: log - groups: - OCSF - name: HTTP Response Code + name: Response Code path: ocsf.http_response.code source: log + type: integer - groups: - OCSF name: HTTP Response Message path: ocsf.http_response.message source: log - - groups: - - OCSF - name: Is Alert - path: ocsf.is_alert - source: log - groups: - OCSF name: Message path: ocsf.message source: log - - groups: - - OCSF - name: Metadata Correlation UID - path: ocsf.metadata.correlation_uid - source: log - - groups: - - OCSF - name: Metadata Log Provider - path: ocsf.metadata.log_provider - source: log - groups: - OCSF name: Metadata Original Time @@ -712,7 +688,7 @@ facets: source: log - groups: - OCSF - name: Metadata UID + name: Metadata Event UID path: ocsf.metadata.uid source: log - groups: @@ -722,34 +698,39 @@ facets: source: log - groups: - OCSF - name: Query Type + name: Query Resource Record Type path: ocsf.query.type source: log - groups: - OCSF - name: Rcode + name: Response Code path: ocsf.rcode source: log - groups: - OCSF - name: Rcode ID + name: Response Code ID path: ocsf.rcode_id source: log - - groups: + type: integer + - facetType: range + groups: - OCSF name: Src Endpoint Port path: ocsf.src_endpoint.port source: log + type: integer - groups: - OCSF name: Status Detail path: ocsf.status_detail source: log - - groups: + - facetType: range + groups: - OCSF - name: Time + name: Event Time path: ocsf.time source: log + type: integer - groups: - OCSF name: TLS Cipher @@ -1714,6 +1695,7 @@ pipeline: target: ocsf.time targetType: attribute preserveSource: false + overrideOnConflict: false - type: pipeline name: OCSF sub pipeline for class Detection Finding [2004] - Notice enabled: true @@ -1766,6 +1748,7 @@ pipeline: target: ocsf.evidence.dst_endpoint.ip targetType: attribute preserveSource: true + overrideOnConflict: false - type: attribute-remapper name: Map `id.resp_p` to `ocsf.evidence.dst_endpoint.port` enabled: true @@ -1775,6 +1758,7 @@ pipeline: target: ocsf.evidence.dst_endpoint.port targetType: attribute preserveSource: true + overrideOnConflict: false - type: array-processor name: Move ocsf.evidence into ocsf.evidences array enabled: true @@ -3245,7 +3229,7 @@ pipeline: overrideOnConflict: true targetFormat: integer - type: schema-remapper - name: Map `query, rcode_name` to `ocsf.message` + name: Map `query`, `rcode_name` to `ocsf.message` sources: - query - rcode_name @@ -3558,7 +3542,7 @@ pipeline: preserveSource: true overrideOnConflict: true - type: schema-remapper - name: Map `seen_bytes, total_bytes` to `ocsf.file.size` + name: Map `seen_bytes`, `total_bytes` to `ocsf.file.size` sources: - seen_bytes - total_bytes From 048277b1ca93a33ab4314032c256515734d4fe11 Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Fri, 15 May 2026 08:42:11 -0500 Subject: [PATCH 03/10] Fix severity mapping for Detection Finding [2004] Notice Notice events emit `severity.name` capitalized ("High", "Medium", etc.), so the lowercase `@severity.name:informational` filters never matched and the fallback assigned `ocsf.severity_id: 99` while preserving the capitalized name as `ocsf.severity`. Switch the schema-category-mapper to filter on the numeric `severity.id` (1-5) which Corelight reliably emits, and update the notice fixture's expected `severity_id` from 99 to 4 to reflect the corrected mapping. Co-Authored-By: Claude Opus 4.7 (1M context) --- zeek/assets/logs/zeek.yaml | 18 +++++++----------- zeek/assets/logs/zeek_tests.yaml | 2 +- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index 8a661babfcde6..8dae696adafa0 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -1858,29 +1858,25 @@ pipeline: name: ocsf.severity_id categories: - filter: - query: "@severity.name:informational" + query: "@severity.id:1" name: Informational id: 1 - filter: - query: "@severity.name:low" + query: "@severity.id:2" name: Low id: 2 - filter: - query: "@severity.name:medium" + query: "@severity.id:3" name: Medium id: 3 - filter: - query: "@severity.name:high" + query: "@severity.id:4" name: High id: 4 - filter: - query: "@severity.name:critical" + query: "@severity.id:5" name: Critical id: 5 - - filter: - query: "@severity.name:*" - name: Other - id: 99 targets: name: ocsf.severity id: ocsf.severity_id @@ -1889,8 +1885,8 @@ pipeline: ocsf.severity: Other ocsf.severity_id: "99" sources: - ocsf.severity: - - severity.name + ocsf.severity_id: + - severity.id - type: schema-category-mapper name: ocsf.status_id categories: diff --git a/zeek/assets/logs/zeek_tests.yaml b/zeek/assets/logs/zeek_tests.yaml index 186c0bc8d548c..9f287fdf6b720 100644 --- a/zeek/assets/logs/zeek_tests.yaml +++ b/zeek/assets/logs/zeek_tests.yaml @@ -1247,7 +1247,7 @@ tests: class_name: Detection Finding status: New severity: High - severity_id: 99 + severity_id: 4 _system_name: test-system actions: - Notice::ACTION_LOG From 98ce4beb7ff66bd910dc5bd160958f56132e2de6 Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Fri, 15 May 2026 09:42:30 -0500 Subject: [PATCH 04/10] Add catch-all category to schema-category-mappers with fallback Each schema-category-mapper that defines a fallback must also have a catch-all filter category at the end matching the fallback's values. Six mappers were missing the trailing catch-all: notice/alert severity_id (2004), http activity_id/status_id (4002), dns rcode_id, and dns status_id (4003). Append `query: "*"` -> Other/99 to each. Co-Authored-By: Claude Opus 4.7 (1M context) --- zeek/assets/logs/zeek.yaml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index 8dae696adafa0..82627890aaa6d 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -1877,6 +1877,10 @@ pipeline: query: "@severity.id:5" name: Critical id: 5 + - filter: + query: "*" + name: Other + id: 99 targets: name: ocsf.severity id: ocsf.severity_id @@ -2167,6 +2171,10 @@ pipeline: query: "@alert.severity:3" name: Low id: 2 + - filter: + query: "*" + name: Other + id: 99 targets: name: ocsf.severity id: ocsf.severity_id @@ -2917,6 +2925,10 @@ pipeline: query: "@http.method:TRACE" name: Trace id: 8 + - filter: + query: "*" + name: Other + id: 99 targets: name: ocsf.activity_name id: ocsf.activity_id @@ -3130,6 +3142,10 @@ pipeline: query: "@http.status_code:[400 TO 599]" name: Failure id: 2 + - filter: + query: "*" + name: Other + id: 99 targets: name: ocsf.status id: ocsf.status_id @@ -3364,6 +3380,10 @@ pipeline: query: "@rcode:23" name: BADCOOKIE id: 23 + - filter: + query: "*" + name: Other + id: 99 targets: name: ocsf.rcode id: ocsf.rcode_id @@ -3417,6 +3437,10 @@ pipeline: query: "(@rcode:3 OR @dns.flags.rcode:NXDOMAIN OR @rejected:true)" name: Failure id: 2 + - filter: + query: "*" + name: Other + id: 99 targets: name: ocsf.status id: ocsf.status_id @@ -3612,9 +3636,20 @@ pipeline: query: "@timedout:true" name: Failure id: 2 + - filter: + query: "@timedout:*" + name: Other + id: 99 targets: name: ocsf.status id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: 99 + sources: + ocsf.status: + - timedout - type: schema-remapper name: Map `ocsf.time` to `ocsf.time` sources: From 7a026ebacdd5fbb292932281625dbf39f491042a Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Fri, 15 May 2026 11:27:47 -0500 Subject: [PATCH 05/10] Apply PR review feedback for Zeek/Corelight OCSF pipeline Direct mappings, dead-code removal, correctness fixes, and OCSF validator cleanups across notice, suricata, conn, ssl, weird, http, dns, and file hosting sub-pipelines: - Map directly to OCSF targets where intermediates were unnecessary (ocsf.time, ocsf.duration, ocsf.traffic.packets, JA3/JA3S algorithm_id, weird protocol_name). - Drop dead/auto-generated mappers: notice/suricata category_uid (set by schema-processor), self-maps of finding_info.uid, event_code, file.hashes (when unbuilt upstream), suricata community_id correlation_uid, HTTP version-as-protocol_ver, DNS direction derivation, and the DNS rcode_id catch-all/fallback (recommended-not-required). - Convert suricata alert.signature_id event_code from string-builder to schema-remapper. - Combine domain/query into single ocsf.query.hostname schema-remapper. - Fix DNS Activity filters: use rcode_name presence to discriminate Response/Query instead of dns.answer.name (handles NXDOMAIN responses). - DNS status_id catch-all renamed Other/99 -> Unknown/0 to satisfy the OCSF validator's suspicious-Other check. - File Hosting tx_hosts/rx_hosts: drop the second intermediate field; grok targets ocsf.{src,dst}_endpoint.ip directly off a single stringify. - Switch fallback source fields per Jonah's suggestions: severity -> severity.name, alert.severity -> alert_severity, http status -> status_msg, dns rcode/status -> rcode_name. - Notice fixture: use id.orig_h/id.resp_h connection fields instead of the suricata-style src. Regenerated zeek_tests.yaml with the OCSF validator (--check-all --write). All 14 logs pass validation with no errors or warnings. Co-Authored-By: Claude Opus 4.7 (1M context) --- zeek/assets/logs/zeek.yaml | 188 +++++++------------------------ zeek/assets/logs/zeek_tests.yaml | 72 +++++++----- 2 files changed, 85 insertions(+), 175 deletions(-) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index 82627890aaa6d..e50a792254be2 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -1685,17 +1685,7 @@ pipeline: - "2023-12-12T05:52:32.763303Z" grok: supportRules: "" - matchRules: 'parse_ts %{date("yyyy-MM-dd''T''HH:mm:ss.SSSSSSZ"):_time_ms}' - - type: attribute-remapper - name: Map `_time_ms` to `ocsf.time` - enabled: true - sources: - - _time_ms - sourceType: attribute - target: ocsf.time - targetType: attribute - preserveSource: false - overrideOnConflict: false + matchRules: 'parse_ts %{date("yyyy-MM-dd''T''HH:mm:ss.SSSSSSZ"):ocsf.time}' - type: pipeline name: OCSF sub pipeline for class Detection Finding [2004] - Notice enabled: true @@ -1781,16 +1771,6 @@ pipeline: targets: name: ocsf.activity_name id: ocsf.activity_id - - type: schema-category-mapper - name: ocsf.category_uid - categories: - - filter: - query: "*" - name: Findings - id: 2 - targets: - name: ocsf.category_name - id: ocsf.category_uid - type: schema-remapper name: Map `ocsf.evidences` to `ocsf.evidences` sources: @@ -1812,13 +1792,6 @@ pipeline: target: ocsf.finding_info.uid preserveSource: true overrideOnConflict: true - - type: schema-remapper - name: Map `note` to `ocsf.finding_info.uid` - sources: - - note - target: ocsf.finding_info.uid - preserveSource: true - overrideOnConflict: true - type: schema-remapper name: Map `ocsf.is_alert` to `ocsf.is_alert` sources: @@ -1889,8 +1862,8 @@ pipeline: ocsf.severity: Other ocsf.severity_id: "99" sources: - ocsf.severity_id: - - severity.id + ocsf.severity: + - severity.name - type: schema-category-mapper name: ocsf.status_id categories: @@ -1936,12 +1909,6 @@ pipeline: template: Corelight target: ocsf.metadata.log_provider replaceMissing: false - - type: string-builder-processor - name: Map alert.signature_id to ocsf.metadata.event_code - enabled: true - template: "%{alert.signature_id}" - target: ocsf.metadata.event_code - replaceMissing: true - type: string-builder-processor name: Set is_alert to boolean true enabled: true @@ -2019,16 +1986,6 @@ pipeline: targets: name: ocsf.activity_name id: ocsf.activity_id - - type: schema-category-mapper - name: ocsf.category_uid - categories: - - filter: - query: "*" - name: Findings - id: 2 - targets: - name: ocsf.category_name - id: ocsf.category_uid - type: schema-category-mapper name: ocsf.confidence_id categories: @@ -2086,13 +2043,6 @@ pipeline: target: ocsf.finding_info.title preserveSource: true overrideOnConflict: true - - type: schema-remapper - name: Map `ocsf.finding_info.uid` to `ocsf.finding_info.uid` - sources: - - ocsf.finding_info.uid - target: ocsf.finding_info.uid - preserveSource: true - overrideOnConflict: true - type: schema-remapper name: Map `uid` to `ocsf.finding_info.uid` sources: @@ -2115,19 +2065,13 @@ pipeline: preserveSource: true overrideOnConflict: true - type: schema-remapper - name: Map `community_id` to `ocsf.metadata.correlation_uid` + name: Map `alert.signature_id` to `ocsf.metadata.event_code` sources: - - community_id - target: ocsf.metadata.correlation_uid - preserveSource: true - overrideOnConflict: true - - type: schema-remapper - name: Map `ocsf.metadata.event_code` to `ocsf.metadata.event_code` - sources: - - ocsf.metadata.event_code + - alert.signature_id target: ocsf.metadata.event_code preserveSource: true overrideOnConflict: true + targetFormat: string - type: schema-remapper name: Map `ocsf.metadata.log_provider` to `ocsf.metadata.log_provider` sources: @@ -2184,7 +2128,7 @@ pipeline: ocsf.severity_id: "99" sources: ocsf.severity: - - alert.severity + - alert_severity - type: schema-remapper name: Map `alert.action` to `ocsf.status_detail` sources: @@ -2225,13 +2169,13 @@ pipeline: name: Calculate total packets enabled: true expression: (orig_pkts + resp_pkts) - target: _total_packets + target: ocsf.traffic.packets isReplaceMissing: false - type: arithmetic-processor name: Convert duration to milliseconds enabled: true expression: duration * 1000 - target: _duration_ms + target: ocsf.duration isReplaceMissing: false - type: schema-processor name: Apply OCSF schema for 4001 @@ -2354,11 +2298,11 @@ pipeline: overrideOnConflict: true targetFormat: integer - type: schema-remapper - name: Map `_duration_ms` to `ocsf.duration` + name: Map `ocsf.duration` to `ocsf.duration` sources: - - _duration_ms + - ocsf.duration target: ocsf.duration - preserveSource: false + preserveSource: true overrideOnConflict: true targetFormat: integer - type: schema-remapper @@ -2480,11 +2424,11 @@ pipeline: overrideOnConflict: true targetFormat: integer - type: schema-remapper - name: Map `_total_packets` to `ocsf.traffic.packets` + name: Map `ocsf.traffic.packets` to `ocsf.traffic.packets` sources: - - _total_packets + - ocsf.traffic.packets target: ocsf.traffic.packets - preserveSource: false + preserveSource: true overrideOnConflict: true targetFormat: integer - type: schema-remapper @@ -2528,7 +2472,7 @@ pipeline: name: Set JA3 hash algorithm id enabled: true template: "1" - target: _ja3_algorithm_id + target: ocsf.tls.ja3_hash.algorithm_id replaceMissing: false - type: string-builder-processor name: Set JA3S hash algorithm name @@ -2540,7 +2484,7 @@ pipeline: name: Set JA3S hash algorithm id enabled: true template: "1" - target: _ja3s_algorithm_id + target: ocsf.tls.ja3s_hash.algorithm_id replaceMissing: false - type: schema-processor name: Apply OCSF schema for 4001 @@ -2672,11 +2616,11 @@ pipeline: preserveSource: true overrideOnConflict: true - type: schema-remapper - name: Map `_ja3_algorithm_id` to `ocsf.tls.ja3_hash.algorithm_id` + name: Map `ocsf.tls.ja3_hash.algorithm_id` to `ocsf.tls.ja3_hash.algorithm_id` sources: - - _ja3_algorithm_id + - ocsf.tls.ja3_hash.algorithm_id target: ocsf.tls.ja3_hash.algorithm_id - preserveSource: false + preserveSource: true overrideOnConflict: true targetFormat: integer - type: schema-remapper @@ -2694,11 +2638,11 @@ pipeline: preserveSource: true overrideOnConflict: true - type: schema-remapper - name: Map `_ja3s_algorithm_id` to `ocsf.tls.ja3s_hash.algorithm_id` + name: Map `ocsf.tls.ja3s_hash.algorithm_id` to `ocsf.tls.ja3s_hash.algorithm_id` sources: - - _ja3s_algorithm_id + - ocsf.tls.ja3s_hash.algorithm_id target: ocsf.tls.ja3s_hash.algorithm_id - preserveSource: false + preserveSource: true overrideOnConflict: true targetFormat: integer - type: schema-remapper @@ -2737,12 +2681,6 @@ pipeline: filter: query: "@_path:weird_red" processors: - - type: string-builder-processor - name: Lowercase source for protocol - enabled: true - template: "%{source}" - target: _protocol_name - replaceMissing: false - type: schema-processor name: Apply OCSF schema for 4001 enabled: true @@ -2768,9 +2706,9 @@ pipeline: name: ocsf.connection_info.direction id: ocsf.connection_info.direction_id - type: schema-remapper - name: Map `_protocol_name` to `ocsf.connection_info.protocol_name` + name: Map `source` to `ocsf.connection_info.protocol_name` sources: - - _protocol_name + - source target: ocsf.connection_info.protocol_name preserveSource: true overrideOnConflict: true @@ -2949,13 +2887,6 @@ pipeline: targets: name: ocsf.connection_info.direction id: ocsf.connection_info.direction_id - - type: schema-remapper - name: Map `version` to `ocsf.connection_info.protocol_ver` - sources: - - version - target: ocsf.connection_info.protocol_ver - preserveSource: true - overrideOnConflict: true - type: schema-remapper name: Map `uid` to `ocsf.connection_info.uid` sources: @@ -3155,7 +3086,7 @@ pipeline: ocsf.status_id: "99" sources: ocsf.status: - - http.status_code + - status_msg - type: schema-remapper name: Map `ocsf.time` to `ocsf.time` sources: @@ -3187,11 +3118,11 @@ pipeline: name: ocsf.activity_id categories: - filter: - query: "@dns.answer.name:*" + query: "@rcode_name:*" name: Response id: 2 - filter: - query: "-@dns.answer.name:*" + query: "-@rcode_name:*" name: Query id: 1 targets: @@ -3201,13 +3132,9 @@ pipeline: name: ocsf.connection_info.direction_id categories: - filter: - query: "@dns.answer.name:*" - name: Inbound - id: 1 - - filter: - query: "-@dns.answer.name:*" - name: Outbound - id: 2 + query: "*" + name: Unknown + id: 0 targets: name: ocsf.connection_info.direction id: ocsf.connection_info.direction_id @@ -3277,8 +3204,9 @@ pipeline: preserveSource: true overrideOnConflict: true - type: schema-remapper - name: Map `query` to `ocsf.query.hostname` + name: Map `domain`, `query` to `ocsf.query.hostname` sources: + - domain - query target: ocsf.query.hostname preserveSource: true @@ -3290,13 +3218,6 @@ pipeline: target: ocsf.query.type preserveSource: true overrideOnConflict: true - - type: schema-remapper - name: Map `rcode_name` to `ocsf.rcode` - sources: - - rcode_name - target: ocsf.rcode - preserveSource: true - overrideOnConflict: true - type: schema-category-mapper name: ocsf.rcode_id categories: @@ -3380,20 +3301,9 @@ pipeline: query: "@rcode:23" name: BADCOOKIE id: 23 - - filter: - query: "*" - name: Other - id: 99 targets: name: ocsf.rcode id: ocsf.rcode_id - fallback: - values: - ocsf.rcode: Other - ocsf.rcode_id: "99" - sources: - ocsf.rcode: - - rcode - type: schema-category-mapper name: ocsf.severity_id categories: @@ -3439,18 +3349,11 @@ pipeline: id: 2 - filter: query: "*" - name: Other - id: 99 + name: Unknown + id: 0 targets: name: ocsf.status id: ocsf.status_id - fallback: - values: - ocsf.status: Other - ocsf.status_id: "99" - sources: - ocsf.status: - - dns.flags.rcode - type: schema-remapper name: Map `ocsf.time` to `ocsf.time` sources: @@ -3494,7 +3397,7 @@ pipeline: - '["10.104.10.60"]' grok: supportRules: "" - matchRules: 'g \[?"?%{ip:_tx_host}"?' + matchRules: 'g \[?"?%{ip:ocsf.src_endpoint.ip}"?' - type: grok-parser name: Extract first IP from rx_hosts enabled: true @@ -3503,7 +3406,7 @@ pipeline: - '["10.104.10.65"]' grok: supportRules: "" - matchRules: 'g \[?"?%{ip:_rx_host}"?' + matchRules: 'g \[?"?%{ip:ocsf.dst_endpoint.ip}"?' - type: schema-processor name: Apply OCSF schema for 6006 enabled: true @@ -3520,7 +3423,7 @@ pipeline: name: Download id: 2 - filter: - query: "-@is_orig:*" + query: "*" name: Unknown id: 0 targets: @@ -3534,19 +3437,12 @@ pipeline: preserveSource: true overrideOnConflict: true - type: schema-remapper - name: Map `_rx_host` to `ocsf.dst_endpoint.ip` + name: Map `ocsf.dst_endpoint.ip` to `ocsf.dst_endpoint.ip` sources: - - _rx_host + - ocsf.dst_endpoint.ip target: ocsf.dst_endpoint.ip preserveSource: true overrideOnConflict: true - - type: schema-remapper - name: Map `ocsf.file.hashes` to `ocsf.file.hashes` - sources: - - ocsf.file.hashes - target: ocsf.file.hashes - preserveSource: true - overrideOnConflict: true - type: schema-remapper name: Map `mime_type` to `ocsf.file.mime_type` sources: @@ -3619,9 +3515,9 @@ pipeline: name: ocsf.severity id: ocsf.severity_id - type: schema-remapper - name: Map `_tx_host` to `ocsf.src_endpoint.ip` + name: Map `ocsf.src_endpoint.ip` to `ocsf.src_endpoint.ip` sources: - - _tx_host + - ocsf.src_endpoint.ip target: ocsf.src_endpoint.ip preserveSource: true overrideOnConflict: true diff --git a/zeek/assets/logs/zeek_tests.yaml b/zeek/assets/logs/zeek_tests.yaml index 9f287fdf6b720..0e11fa1af3403 100644 --- a/zeek/assets/logs/zeek_tests.yaml +++ b/zeek/assets/logs/zeek_tests.yaml @@ -237,7 +237,6 @@ tests: status_id: 1 connection_info: uid: CBQBvs1hob384mp2lb - protocol_ver: '1.1' direction_id: 0 direction: Unknown class_uid: 4002 @@ -404,9 +403,9 @@ tests: status_id: 1 connection_info: uid: CsOSdHqRMu62rNs31 - direction_id: 1 protocol_name: udp - direction: Inbound + direction_id: 0 + direction: Unknown class_uid: 4003 activity_id: 2 time: 1702360352763 @@ -549,8 +548,8 @@ tests: message: time.missouri.edu connection_info: uid: CyArTY2KEgcygwMLi1 - direction_id: 2 - direction: Outbound + direction_id: 0 + direction: Unknown class_uid: 4003 activity_id: 1 time: 1709786320085 @@ -559,6 +558,8 @@ tests: ip: 20.20.20.20 severity_id: 1 class_name: DNS Activity + status_id: 0 + status: Unknown qtype_name: A query: time.missouri.edu id: @@ -837,7 +838,6 @@ tests: status_id: 1 connection_info: uid: CBQBvs1hob384mp2lb - protocol_ver: '1.1' direction_id: 0 direction: Unknown class_uid: 4002 @@ -964,9 +964,9 @@ tests: status_id: 1 connection_info: uid: CsOSdHqRMu62rNs31 - direction_id: 1 protocol_name: udp - direction: Inbound + direction_id: 0 + direction: Unknown class_uid: 4003 activity_id: 2 time: 1702360352763 @@ -1205,25 +1205,29 @@ tests: resp_p: 53 ts: '2026-05-11T16:11:53.944496Z' notice: false - _protocol_name: DNS service: weird_red message: <134>May 11 16:11:53 test-system {"_path":"weird_red","_system_name":"test-system","_write_ts":"2026-05-11T16:11:53.944496Z","ts":"2026-05-11T16:11:53.944496Z","uid":"CYThCC2DZNVcGgpiI2","id.orig_h":"10.250.182.4","id.orig_p":58438,"id.resp_h":"180.153.200.12","id.resp_p":53,"id.vlan":1600,"name":"DNS_truncated_quest_too_short","source":"DNS","notice":false,"peer":"worker-04"} tags: - source:LOGS_SOURCE timestamp: 1778515913944 - - sample: "<134>May 11 19:11:03 test-system {\"_path\":\"notice\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T19:11:03.521194Z\",\"ts\":\"2026-05-11T19:11:03.521194Z\",\"note\":\"ATTACK::Discovery\",\"msg\":\"Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins\",\"src\":\"10.10.10.15\",\"peer_descr\":\"manager\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600, \"uid\":\"12345678901234567890\", \"severity\":{\"name\":\"High\",\"id\":4}}" + sample: "<134>May 11 19:11:03 test-system {\"_path\":\"notice\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T19:11:03.521194Z\",\"ts\":\"2026-05-11T19:11:03.521194Z\",\"note\":\"ATTACK::Discovery\",\"msg\":\"Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins\",\"id.orig_h\":\"10.10.10.15\",\"id.orig_p\":54321,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":80,\"peer_descr\":\"manager\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600, \"uid\":\"12345678901234567890\", \"severity\":{\"name\":\"High\",\"id\":4}}" service: "corelight" result: custom: msg: Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins suppress_for: 3600 - src: 10.10.10.15 _write_ts: '2026-05-11T19:11:03.521194Z' + network: + client: + ip: 10.10.10.15 + port: 54321 + destination: + ip: 192.168.1.1 + port: 80 zeek: note: ATTACK::Discovery _path: notice - _is_alert_str: 'true' peer_descr: manager ocsf: activity_name: Create @@ -1231,23 +1235,30 @@ tests: product: name: Zeek vendor_name: Corelight - event_code: ATTACK::Discovery version: 1.5.0 uid: '12345678901234567890' - category_uid: 2 - category_name: Findings + event_code: ATTACK::Discovery is_alert: true finding_info: - uid: ATTACK::Discovery + uid: '12345678901234567890' title: ATTACK::Discovery status_id: 1 - class_uid: 2004 - activity_id: 1 time: 1778526663521 - class_name: Detection Finding status: New severity: High severity_id: 4 + evidences: + - src_endpoint: + ip: 10.10.10.15 + port: 54321 + dst_endpoint: + ip: 192.168.1.1 + port: 80 + category_uid: 2 + category_name: Findings + class_uid: 2004 + activity_id: 1 + class_name: Detection Finding _system_name: test-system actions: - Notice::ACTION_LOG @@ -1257,8 +1268,14 @@ tests: severity: name: High id: 4 + _is_alert_str: 'true' + id: + orig_p: 54321 + resp_h: 192.168.1.1 + orig_h: 10.10.10.15 + resp_p: 80 service: notice - message: <134>May 11 19:11:03 test-system {"_path":"notice","_system_name":"test-system","_write_ts":"2026-05-11T19:11:03.521194Z","ts":"2026-05-11T19:11:03.521194Z","note":"ATTACK::Discovery","msg":"Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins","src":"10.10.10.15","peer_descr":"manager","actions":["Notice::ACTION_LOG"],"suppress_for":3600, "uid":"12345678901234567890", "severity":{"name":"High","id":4}} + message: <134>May 11 19:11:03 test-system {"_path":"notice","_system_name":"test-system","_write_ts":"2026-05-11T19:11:03.521194Z","ts":"2026-05-11T19:11:03.521194Z","note":"ATTACK::Discovery","msg":"Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins","id.orig_h":"10.10.10.15","id.orig_p":54321,"id.resp_h":"192.168.1.1","id.resp_p":80,"peer_descr":"manager","actions":["Notice::ACTION_LOG"],"suppress_for":3600, "uid":"12345678901234567890", "severity":{"name":"High","id":4}} tags: - source:LOGS_SOURCE timestamp: 1778526663521 @@ -1292,7 +1309,6 @@ tests: name: Suricata vendor_name: Corelight event_code: '2006380' - correlation_uid: 1:Y4yC2kKf1Cm0QUkA/aBLSDxepcY= log_provider: Corelight version: 1.5.0 category_uid: 2 @@ -1380,8 +1396,6 @@ tests: version: 1.5.0 category_uid: 6 category_name: Application Activity - src_endpoint: - ip: 10.104.10.60 actor: session: uid: CjTuQU17IDvaVa8Nq2 @@ -1395,11 +1409,13 @@ tests: class_uid: 6006 activity_id: 2 time: 1778527585875 - dst_endpoint: - ip: 10.104.10.65 severity_id: 1 class_name: File Hosting Activity status: Success + src_endpoint: + ip: 10.104.10.60 + dst_endpoint: + ip: 10.104.10.65 fuid: FOPDsn3PdkiZsljcj2 id: vlan: 1010 @@ -1428,13 +1444,11 @@ tests: - CjTuQU17IDvaGb8Nq2 ts: '2026-05-11T19:26:25.875206Z' md5: 6e6ae0ed19f595687684faafae5499e13 - _tx_hosts_str: 10.104.10.60 - _rx_hosts_str: 10.104.10.65 - _rx_host: 10.104.10.65 - _tx_host: 10.104.10.60 total_bytes: 253109 seen_bytes: 253109 missing_bytes: 0 + _tx_hosts_str: 10.104.10.60 + _rx_hosts_str: 10.104.10.65 service: files message: <134>May 11 19:26:26 ndr-dub-stryker-DC-1 {"_path":"files","_system_name":"ndr-dub-stryker-DC-1","_write_ts":"2026-05-11T19:26:26.082433Z","ts":"2026-05-11T19:26:25.875206Z","uid":"CjTuQU17IDvaVa8Nq2","fuid":"FOPDsn3PdkiZsljcj2","tx_hosts":["10.104.10.60"],"rx_hosts":["10.104.10.65"],"conn_uids":["CjTuQU17IDvaGb8Nq2"],"source":"HTTP","depth":0,"analyzers":["SHA1","MD5","SHA256","DATA_EVENT"],"local_orig":true,"is_orig":false,"seen_bytes":253109,"total_bytes":253109,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"duration":0.2072269916534424,"mime_type":"text/json","md5":"6e6ae0ed19f595687684faafae5499e13","sha1":"f6578daa6d398c91398888b91a96d4c0e099c79c","sha256":"a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f","id.vlan":1010} tags: From b358fb3ab5695ab6f2a93f8dd60cbb938d1d571f Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Fri, 15 May 2026 12:06:23 -0500 Subject: [PATCH 06/10] Map Zeek DNS answers to ocsf.answers as dns_answer objects Use two array-processors to wrap each Zeek `answers` string into a dns_answer object and append to ocsf.answers: the first selects the first array element into ocsf.answer.rdata, the second appends ocsf.answer onto ocsf.answers. Only the first answer is captured (the pipeline DSL has no per-element iteration), but that covers the common single-A-record case. Co-Authored-By: Claude Opus 4.7 (1M context) --- zeek/assets/logs/zeek.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index e50a792254be2..1774bca18a426 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -3110,6 +3110,21 @@ pipeline: filter: query: "@_path:(dns OR dns_red)" processors: + - type: array-processor + name: Select first answer IP into ocsf.answer.rdata + enabled: true + operation: + source: answers + target: ocsf.answer.rdata + type: select + - type: array-processor + name: Append ocsf.answer into ocsf.answers array + enabled: true + operation: + source: ocsf.answer + target: ocsf.answers + preserveSource: false + type: append - type: schema-processor name: Apply OCSF schema for 4003 enabled: true @@ -3218,6 +3233,13 @@ pipeline: target: ocsf.query.type preserveSource: true overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.answers` to `ocsf.answers` + sources: + - ocsf.answers + target: ocsf.answers + preserveSource: true + overrideOnConflict: true - type: schema-category-mapper name: ocsf.rcode_id categories: From c6a6695f6bd7d96a73d57590fca0455fd3611ac2 Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Fri, 15 May 2026 12:13:04 -0500 Subject: [PATCH 07/10] Add catch-all for activity_id --- zeek/assets/logs/zeek.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index 1774bca18a426..e064fed286b8c 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -2501,6 +2501,10 @@ pipeline: query: "@established:false" name: Fail id: 4 + - filter: + query: "*" + name: Unknown + id: 0 targets: name: ocsf.activity_name id: ocsf.activity_id From 501511a3edb92cbc54990109507809b07d91262b Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Mon, 18 May 2026 09:04:41 -0500 Subject: [PATCH 08/10] Fix validate-logs failure for DNS answers wrapper The previous array-processor type:select required operation.filter and operation.valueToExtract per the asset validator, but those only apply to object arrays - Zeek's `answers` is a primitive string array. Switch to string-builder + grok-parser to extract the first answer string into ocsf.answer.rdata, then keep the array-processor append to wrap it into ocsf.answers as a dns_answer object. Co-Authored-By: Claude Opus 4.7 (1M context) --- zeek/assets/logs/zeek.yaml | 20 ++++++++++++++------ zeek/assets/logs/zeek_tests.yaml | 6 ++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index e064fed286b8c..3cb0b50d482bd 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -3114,13 +3114,21 @@ pipeline: filter: query: "@_path:(dns OR dns_red)" processors: - - type: array-processor - name: Select first answer IP into ocsf.answer.rdata + - type: string-builder-processor + name: Stringify answers enabled: true - operation: - source: answers - target: ocsf.answer.rdata - type: select + template: "%{answers}" + target: _answers_str + replaceMissing: false + - type: grok-parser + name: Extract first answer into ocsf.answer.rdata + enabled: true + source: _answers_str + samples: + - '["185.64.148.0"]' + grok: + supportRules: "" + matchRules: 'a \[?"?%{notSpace:ocsf.answer.rdata}"?' - type: array-processor name: Append ocsf.answer into ocsf.answers array enabled: true diff --git a/zeek/assets/logs/zeek_tests.yaml b/zeek/assets/logs/zeek_tests.yaml index 0e11fa1af3403..669a9f2bd1a19 100644 --- a/zeek/assets/logs/zeek_tests.yaml +++ b/zeek/assets/logs/zeek_tests.yaml @@ -415,6 +415,8 @@ tests: severity_id: 1 class_name: DNS Activity status: Success + answers: + - rdata: 185.64.148.0 query: win2k16-1-159 answers: - 185.64.148.0 @@ -426,6 +428,7 @@ tests: resp_h: 185.64.148.0 orig_h: 185.64.148.0 resp_p: 5355 + _answers_str: 185.64.148.0 message: |- { "AA" : false, @@ -976,6 +979,8 @@ tests: severity_id: 1 class_name: DNS Activity status: Success + answers: + - rdata: 185.64.148.0 Z: 0 ts: '2023-12-12T05:52:32.763303Z' query: win2k16-1-159 @@ -989,6 +994,7 @@ tests: resp_h: 185.64.148.0 orig_h: 185.64.148.0 resp_p: 5355 + _answers_str: 185.64.148.0 service: dns message: <134>Dec 12 05:52:50 machine-name {"_path":"dns","_write_ts":"2023-12-12T05:52:50.756358Z","ts":"2023-12-12T05:52:32.763303Z","uid":"CsOSdHqRMu62rNs31","id.orig_h":"185.64.148.0","id.orig_p":58013,"id.resp_h":"185.64.148.0","id.resp_p":5355,"proto":"udp","trans_id":38706,"rcode":0,"rcode_name":"NOERROR","query":"win2k16-1-159","answers":["185.64.148.0"],"TTLs":[30.0],"AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} tags: From d59cf2ad4d9357166ef9e31487a821b7fcc084fb Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Mon, 18 May 2026 09:33:01 -0500 Subject: [PATCH 09/10] Address codex review feedback for file pipeline - Include `files_red` in the File Hosting [6006] sub-pipeline filter so redacted file events get OCSF class_uid/activity_id/file fields, not just the pre-transform metadata. - Prefer `filename` over `fuid` when populating `ocsf.file.name`; fall back to `fuid` only when `filename` is absent. The `fuid` mapping to `ocsf.file.uid` is unaffected. Co-Authored-By: Claude Opus 4.7 (1M context) --- zeek/assets/logs/zeek.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index 3cb0b50d482bd..cd2105f9becb5 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -3409,7 +3409,7 @@ pipeline: ocsf: isOcsf: true filter: - query: "@_path:files" + query: "@_path:(files OR files_red)" processors: - type: string-builder-processor name: Stringify tx_hosts @@ -3485,8 +3485,9 @@ pipeline: preserveSource: true overrideOnConflict: true - type: schema-remapper - name: Map `fuid` to `ocsf.file.name` + name: Map `filename`, `fuid` to `ocsf.file.name` sources: + - filename - fuid target: ocsf.file.name preserveSource: true From 7c4d76c445507a95cb559c8f56c56e32f0aa3b08 Mon Sep 17 00:00:00 2001 From: Crash <86613440+cepolation-datadog@users.noreply.github.com> Date: Mon, 18 May 2026 12:29:11 -0500 Subject: [PATCH 10/10] Drop pipeline intermediates, fix multi-IP grok, restore file.hashes - is_alert (notice 2004, suricata 2004): string-builder writes directly to `ocsf.is_alert`; grok-parser converts in place. Drops the `_is_alert_str` intermediate. - DNS answers: stringify directly into `ocsf.answer`; grok extracts `ocsf.answer.rdata` via `a %{data:ocsf.answer.rdata}(,%{data})?` so the comma-separated multi-IP form parses correctly. Drops the `_answers_str` intermediate. - File Hosting tx/rx hosts: stringify directly into `ocsf.{src,dst}_endpoint`; grok extracts `.ip` via `g %{ip:ocsf.{src,dst}_endpoint.ip}(,%{data})?` for multi-IP. Drops the `_tx_hosts_str`/`_rx_hosts_str` intermediates. - Connection 4001: arithmetic-processor writes total bytes directly to `ocsf.traffic.bytes`; the schema-processor remapper becomes a self-map. Drops the `_total_bytes` intermediate (matches the earlier _total_packets/_duration_ms cleanup). - Restore `ocsf.file.hashes`: build `tmp_md5`/`tmp_sha1`/`tmp_sha256` fingerprint objects (algorithm name, integer algorithm_id, value), array-processor append each into `ocsf.file.hashes`, and self-map the array inside the 6006 schema-processor. Co-Authored-By: Claude Opus 4.7 (1M context) --- zeek/assets/logs/zeek.yaml | 177 ++++++++++++++++++++++++++----- zeek/assets/logs/zeek_tests.yaml | 24 +++-- 2 files changed, 167 insertions(+), 34 deletions(-) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index cd2105f9becb5..bbce300a8559b 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -1698,12 +1698,12 @@ pipeline: name: Set is_alert to boolean true enabled: true template: "true" - target: _is_alert_str + target: ocsf.is_alert replaceMissing: false - type: grok-parser - name: Convert _is_alert_str to boolean ocsf.is_alert + name: Convert ocsf.is_alert string to boolean enabled: true - source: _is_alert_str + source: ocsf.is_alert samples: - "true" grok: @@ -1913,12 +1913,12 @@ pipeline: name: Set is_alert to boolean true enabled: true template: "true" - target: _is_alert_str + target: ocsf.is_alert replaceMissing: false - type: grok-parser - name: Convert _is_alert_str to boolean ocsf.is_alert + name: Convert ocsf.is_alert string to boolean enabled: true - source: _is_alert_str + source: ocsf.is_alert samples: - "true" grok: @@ -2163,7 +2163,7 @@ pipeline: name: Calculate total bytes enabled: true expression: (orig_bytes + resp_bytes) - target: _total_bytes + target: ocsf.traffic.bytes isReplaceMissing: false - type: arithmetic-processor name: Calculate total packets @@ -2392,11 +2392,11 @@ pipeline: overrideOnConflict: true targetFormat: integer - type: schema-remapper - name: Map `_total_bytes` to `ocsf.traffic.bytes` + name: Map `ocsf.traffic.bytes` to `ocsf.traffic.bytes` sources: - - _total_bytes + - ocsf.traffic.bytes target: ocsf.traffic.bytes - preserveSource: false + preserveSource: true overrideOnConflict: true targetFormat: integer - type: schema-remapper @@ -3115,20 +3115,21 @@ pipeline: query: "@_path:(dns OR dns_red)" processors: - type: string-builder-processor - name: Stringify answers + name: Stringify answers into ocsf.answer enabled: true template: "%{answers}" - target: _answers_str + target: ocsf.answer replaceMissing: false - type: grok-parser name: Extract first answer into ocsf.answer.rdata enabled: true - source: _answers_str + source: ocsf.answer samples: - - '["185.64.148.0"]' + - "185.64.148.0" + - "185.64.148.0,8.8.8.8" grok: supportRules: "" - matchRules: 'a \[?"?%{notSpace:ocsf.answer.rdata}"?' + matchRules: 'a %{data:ocsf.answer.rdata}(,%{data})?' - type: array-processor name: Append ocsf.answer into ocsf.answers array enabled: true @@ -3412,35 +3413,154 @@ pipeline: query: "@_path:(files OR files_red)" processors: - type: string-builder-processor - name: Stringify tx_hosts + name: Stringify tx_hosts into ocsf.src_endpoint enabled: true template: "%{tx_hosts}" - target: _tx_hosts_str + target: ocsf.src_endpoint replaceMissing: false - type: string-builder-processor - name: Stringify rx_hosts + name: Stringify rx_hosts into ocsf.dst_endpoint enabled: true template: "%{rx_hosts}" - target: _rx_hosts_str + target: ocsf.dst_endpoint replaceMissing: false - type: grok-parser name: Extract first IP from tx_hosts enabled: true - source: _tx_hosts_str + source: ocsf.src_endpoint samples: - - '["10.104.10.60"]' + - "10.104.10.60" + - "10.104.10.60,10.104.10.61" grok: supportRules: "" - matchRules: 'g \[?"?%{ip:ocsf.src_endpoint.ip}"?' + matchRules: 'g %{ip:ocsf.src_endpoint.ip}(,%{data})?' - type: grok-parser name: Extract first IP from rx_hosts enabled: true - source: _rx_hosts_str + source: ocsf.dst_endpoint + samples: + - "10.104.10.65" + - "10.104.10.65,10.104.10.66" + grok: + supportRules: "" + matchRules: 'g %{ip:ocsf.dst_endpoint.ip}(,%{data})?' + - type: string-builder-processor + name: Set MD5 algorithm name + enabled: true + template: MD5 + target: tmp_md5.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set MD5 algorithm id + enabled: true + template: "1" + target: tmp_md5.algorithm_id + replaceMissing: false + - type: grok-parser + name: Coerce tmp_md5.algorithm_id to integer + enabled: true + source: tmp_md5.algorithm_id samples: - - '["10.104.10.65"]' + - "1" grok: supportRules: "" - matchRules: 'g \[?"?%{ip:ocsf.dst_endpoint.ip}"?' + matchRules: "to_int %{integer:tmp_md5.algorithm_id}" + - type: attribute-remapper + name: Map `md5` to `tmp_md5.value` + enabled: true + sources: + - md5 + sourceType: attribute + target: tmp_md5.value + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Append tmp_md5 to ocsf.file.hashes + enabled: true + operation: + source: tmp_md5 + target: ocsf.file.hashes + preserveSource: false + type: append + - type: string-builder-processor + name: Set SHA1 algorithm name + enabled: true + template: SHA-1 + target: tmp_sha1.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set SHA1 algorithm id + enabled: true + template: "2" + target: tmp_sha1.algorithm_id + replaceMissing: false + - type: grok-parser + name: Coerce tmp_sha1.algorithm_id to integer + enabled: true + source: tmp_sha1.algorithm_id + samples: + - "2" + grok: + supportRules: "" + matchRules: "to_int %{integer:tmp_sha1.algorithm_id}" + - type: attribute-remapper + name: Map `sha1` to `tmp_sha1.value` + enabled: true + sources: + - sha1 + sourceType: attribute + target: tmp_sha1.value + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Append tmp_sha1 to ocsf.file.hashes + enabled: true + operation: + source: tmp_sha1 + target: ocsf.file.hashes + preserveSource: false + type: append + - type: string-builder-processor + name: Set SHA256 algorithm name + enabled: true + template: SHA-256 + target: tmp_sha256.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set SHA256 algorithm id + enabled: true + template: "3" + target: tmp_sha256.algorithm_id + replaceMissing: false + - type: grok-parser + name: Coerce tmp_sha256.algorithm_id to integer + enabled: true + source: tmp_sha256.algorithm_id + samples: + - "3" + grok: + supportRules: "" + matchRules: "to_int %{integer:tmp_sha256.algorithm_id}" + - type: attribute-remapper + name: Map `sha256` to `tmp_sha256.value` + enabled: true + sources: + - sha256 + sourceType: attribute + target: tmp_sha256.value + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Append tmp_sha256 to ocsf.file.hashes + enabled: true + operation: + source: tmp_sha256 + target: ocsf.file.hashes + preserveSource: false + type: append - type: schema-processor name: Apply OCSF schema for 6006 enabled: true @@ -3477,6 +3597,13 @@ pipeline: target: ocsf.dst_endpoint.ip preserveSource: true overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.file.hashes` to `ocsf.file.hashes` + sources: + - ocsf.file.hashes + target: ocsf.file.hashes + preserveSource: true + overrideOnConflict: true - type: schema-remapper name: Map `mime_type` to `ocsf.file.mime_type` sources: diff --git a/zeek/assets/logs/zeek_tests.yaml b/zeek/assets/logs/zeek_tests.yaml index 669a9f2bd1a19..55e841d90bd83 100644 --- a/zeek/assets/logs/zeek_tests.yaml +++ b/zeek/assets/logs/zeek_tests.yaml @@ -311,7 +311,7 @@ tests: "rejected" : false, "query" : "win2k16-1-159", "_write_ts" : "2023-12-12T05:52:50.756358Z", - "answers" : [ "185.64.148.0" ], + "answers" : [ "185.64.148.0", "8.8.8.8" ], "trans_id" : 38706, "rcode" : 0, "_path" : "dns", @@ -344,7 +344,8 @@ tests: dns: answer: name: - - "185.64.148.0" + - 185.64.148.0 + - 8.8.8.8 flags: rcode: "NOERROR" id: 38706 @@ -420,6 +421,7 @@ tests: query: win2k16-1-159 answers: - 185.64.148.0 + - 8.8.8.8 trans_id: 38706 rcode_name: NOERROR proto: udp @@ -428,7 +430,6 @@ tests: resp_h: 185.64.148.0 orig_h: 185.64.148.0 resp_p: 5355 - _answers_str: 185.64.148.0 message: |- { "AA" : false, @@ -436,7 +437,7 @@ tests: "rejected" : false, "query" : "win2k16-1-159", "_write_ts" : "2023-12-12T05:52:50.756358Z", - "answers" : [ "185.64.148.0" ], + "answers" : [ "185.64.148.0", "8.8.8.8" ], "trans_id" : 38706, "rcode" : 0, "_path" : "dns", @@ -994,7 +995,6 @@ tests: resp_h: 185.64.148.0 orig_h: 185.64.148.0 resp_p: 5355 - _answers_str: 185.64.148.0 service: dns message: <134>Dec 12 05:52:50 machine-name {"_path":"dns","_write_ts":"2023-12-12T05:52:50.756358Z","ts":"2023-12-12T05:52:32.763303Z","uid":"CsOSdHqRMu62rNs31","id.orig_h":"185.64.148.0","id.orig_p":58013,"id.resp_h":"185.64.148.0","id.resp_p":5355,"proto":"udp","trans_id":38706,"rcode":0,"rcode_name":"NOERROR","query":"win2k16-1-159","answers":["185.64.148.0"],"TTLs":[30.0],"AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} tags: @@ -1274,7 +1274,6 @@ tests: severity: name: High id: 4 - _is_alert_str: 'true' id: orig_p: 54321 resp_h: 192.168.1.1 @@ -1295,7 +1294,6 @@ tests: _write_ts: '2026-05-11T17:59:59.359532Z' suri_id: SOHaIDWJ5dBe _path: suricata_corelight - _is_alert_str: 'true' tx_id: 0 network: destination: @@ -1411,6 +1409,16 @@ tests: mime_type: text/json type_id: 1 name: FOPDsn3PdkiZsljcj2 + hashes: + - algorithm_id: 1 + value: 6e6ae0ed19f595687684faafae5499e13 + algorithm: MD5 + - algorithm_id: 2 + value: f6578daa6d398c91398888b91a96d4c0e099c79c + algorithm: SHA-1 + - algorithm_id: 3 + value: a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f + algorithm: SHA-256 status_id: 1 class_uid: 6006 activity_id: 2 @@ -1453,8 +1461,6 @@ tests: total_bytes: 253109 seen_bytes: 253109 missing_bytes: 0 - _tx_hosts_str: 10.104.10.60 - _rx_hosts_str: 10.104.10.65 service: files message: <134>May 11 19:26:26 ndr-dub-stryker-DC-1 {"_path":"files","_system_name":"ndr-dub-stryker-DC-1","_write_ts":"2026-05-11T19:26:26.082433Z","ts":"2026-05-11T19:26:25.875206Z","uid":"CjTuQU17IDvaVa8Nq2","fuid":"FOPDsn3PdkiZsljcj2","tx_hosts":["10.104.10.60"],"rx_hosts":["10.104.10.65"],"conn_uids":["CjTuQU17IDvaGb8Nq2"],"source":"HTTP","depth":0,"analyzers":["SHA1","MD5","SHA256","DATA_EVENT"],"local_orig":true,"is_orig":false,"seen_bytes":253109,"total_bytes":253109,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"duration":0.2072269916534424,"mime_type":"text/json","md5":"6e6ae0ed19f595687684faafae5499e13","sha1":"f6578daa6d398c91398888b91a96d4c0e099c79c","sha256":"a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f","id.vlan":1010} tags: