Drop pipeline intermediates, fix multi-IP grok, restore file.hashes

- is_alert (notice 2004, suricata 2004): string-builder writes directly
  to `ocsf.is_alert`; grok-parser converts in place. Drops the
  `_is_alert_str` intermediate.
- DNS answers: stringify directly into `ocsf.answer`; grok extracts
  `ocsf.answer.rdata` via `a %{data:ocsf.answer.rdata}(,%{data})?` so
  the comma-separated multi-IP form parses correctly. Drops the
  `_answers_str` intermediate.
- File Hosting tx/rx hosts: stringify directly into
  `ocsf.{src,dst}_endpoint`; grok extracts `.ip` via
  `g %{ip:ocsf.{src,dst}_endpoint.ip}(,%{data})?` for multi-IP. Drops
  the `_tx_hosts_str`/`_rx_hosts_str` intermediates.
- Connection 4001: arithmetic-processor writes total bytes directly to
  `ocsf.traffic.bytes`; the schema-processor remapper becomes a
  self-map. Drops the `_total_bytes` intermediate (matches the
  earlier _total_packets/_duration_ms cleanup).
- Restore `ocsf.file.hashes`: build `tmp_md5`/`tmp_sha1`/`tmp_sha256`
  fingerprint objects (algorithm name, integer algorithm_id, value),
  array-processor append each into `ocsf.file.hashes`, and self-map
  the array inside the 6006 schema-processor.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: cepolation-datadog

zeek/assets/logs/zeek.yaml

@@ -1698,12 +1698,12 @@ pipeline:
           name: Set is_alert to boolean true
           enabled: true
           template: "true"
-          target: _is_alert_str
+          target: ocsf.is_alert
           replaceMissing: false
         - type: grok-parser
-          name: Convert _is_alert_str to boolean ocsf.is_alert
+          name: Convert ocsf.is_alert string to boolean
           enabled: true
-          source: _is_alert_str
+          source: ocsf.is_alert
           samples:
             - "true"
           grok:
@@ -1913,12 +1913,12 @@ pipeline:
           name: Set is_alert to boolean true
           enabled: true
           template: "true"
-          target: _is_alert_str
+          target: ocsf.is_alert
           replaceMissing: false
         - type: grok-parser
-          name: Convert _is_alert_str to boolean ocsf.is_alert
+          name: Convert ocsf.is_alert string to boolean
           enabled: true
-          source: _is_alert_str
+          source: ocsf.is_alert
           samples:
             - "true"
           grok:
@@ -2163,7 +2163,7 @@ pipeline:
           name: Calculate total bytes
           enabled: true
           expression: (orig_bytes + resp_bytes)
-          target: _total_bytes
+          target: ocsf.traffic.bytes
           isReplaceMissing: false
         - type: arithmetic-processor
           name: Calculate total packets
@@ -2392,11 +2392,11 @@ pipeline:
               overrideOnConflict: true
               targetFormat: integer
             - type: schema-remapper
-              name: Map `_total_bytes` to `ocsf.traffic.bytes`
+              name: Map `ocsf.traffic.bytes` to `ocsf.traffic.bytes`
               sources:
-                - _total_bytes
+                - ocsf.traffic.bytes
               target: ocsf.traffic.bytes
-              preserveSource: false
+              preserveSource: true
               overrideOnConflict: true
               targetFormat: integer
             - type: schema-remapper
@@ -3115,20 +3115,21 @@ pipeline:
         query: "@_path:(dns OR dns_red)"
       processors:
         - type: string-builder-processor
-          name: Stringify answers
+          name: Stringify answers into ocsf.answer
           enabled: true
           template: "%{answers}"
-          target: _answers_str
+          target: ocsf.answer
           replaceMissing: false
         - type: grok-parser
           name: Extract first answer into ocsf.answer.rdata
           enabled: true
-          source: _answers_str
+          source: ocsf.answer
           samples:
-            - '["185.64.148.0"]'
+            - "185.64.148.0"
+            - "185.64.148.0,8.8.8.8"
           grok:
             supportRules: ""
-            matchRules: 'a \[?"?%{notSpace:ocsf.answer.rdata}"?'
+            matchRules: 'a %{data:ocsf.answer.rdata}(,%{data})?'
         - type: array-processor
           name: Append ocsf.answer into ocsf.answers array
           enabled: true
@@ -3412,35 +3413,154 @@ pipeline:
         query: "@_path:(files OR files_red)"
       processors:
         - type: string-builder-processor
-          name: Stringify tx_hosts
+          name: Stringify tx_hosts into ocsf.src_endpoint
           enabled: true
           template: "%{tx_hosts}"
-          target: _tx_hosts_str
+          target: ocsf.src_endpoint
           replaceMissing: false
         - type: string-builder-processor
-          name: Stringify rx_hosts
+          name: Stringify rx_hosts into ocsf.dst_endpoint
           enabled: true
           template: "%{rx_hosts}"
-          target: _rx_hosts_str
+          target: ocsf.dst_endpoint
           replaceMissing: false
         - type: grok-parser
           name: Extract first IP from tx_hosts
           enabled: true
-          source: _tx_hosts_str
+          source: ocsf.src_endpoint
           samples:
-            - '["10.104.10.60"]'
+            - "10.104.10.60"
+            - "10.104.10.60,10.104.10.61"
           grok:
             supportRules: ""
-            matchRules: 'g \[?"?%{ip:ocsf.src_endpoint.ip}"?'
+            matchRules: 'g %{ip:ocsf.src_endpoint.ip}(,%{data})?'
         - type: grok-parser
           name: Extract first IP from rx_hosts
           enabled: true
-          source: _rx_hosts_str
+          source: ocsf.dst_endpoint
+          samples:
+            - "10.104.10.65"
+            - "10.104.10.65,10.104.10.66"
+          grok:
+            supportRules: ""
+            matchRules: 'g %{ip:ocsf.dst_endpoint.ip}(,%{data})?'
+        - type: string-builder-processor
+          name: Set MD5 algorithm name
+          enabled: true
+          template: MD5
+          target: tmp_md5.algorithm
+          replaceMissing: false
+        - type: string-builder-processor
+          name: Set MD5 algorithm id
+          enabled: true
+          template: "1"
+          target: tmp_md5.algorithm_id
+          replaceMissing: false
+        - type: grok-parser
+          name: Coerce tmp_md5.algorithm_id to integer
+          enabled: true
+          source: tmp_md5.algorithm_id
           samples:
-            - '["10.104.10.65"]'
+            - "1"
           grok:
             supportRules: ""
-            matchRules: 'g \[?"?%{ip:ocsf.dst_endpoint.ip}"?'
+            matchRules: "to_int %{integer:tmp_md5.algorithm_id}"
+        - type: attribute-remapper
+          name: Map `md5` to `tmp_md5.value`
+          enabled: true
+          sources:
+            - md5
+          sourceType: attribute
+          target: tmp_md5.value
+          targetType: attribute
+          preserveSource: true
+          overrideOnConflict: false
+        - type: array-processor
+          name: Append tmp_md5 to ocsf.file.hashes
+          enabled: true
+          operation:
+            source: tmp_md5
+            target: ocsf.file.hashes
+            preserveSource: false
+            type: append
+        - type: string-builder-processor
+          name: Set SHA1 algorithm name
+          enabled: true
+          template: SHA-1
+          target: tmp_sha1.algorithm
+          replaceMissing: false
+        - type: string-builder-processor
+          name: Set SHA1 algorithm id
+          enabled: true
+          template: "2"
+          target: tmp_sha1.algorithm_id
+          replaceMissing: false
+        - type: grok-parser
+          name: Coerce tmp_sha1.algorithm_id to integer
+          enabled: true
+          source: tmp_sha1.algorithm_id
+          samples:
+            - "2"
+          grok:
+            supportRules: ""
+            matchRules: "to_int %{integer:tmp_sha1.algorithm_id}"
+        - type: attribute-remapper
+          name: Map `sha1` to `tmp_sha1.value`
+          enabled: true
+          sources:
+            - sha1
+          sourceType: attribute
+          target: tmp_sha1.value
+          targetType: attribute
+          preserveSource: true
+          overrideOnConflict: false
+        - type: array-processor
+          name: Append tmp_sha1 to ocsf.file.hashes
+          enabled: true
+          operation:
+            source: tmp_sha1
+            target: ocsf.file.hashes
+            preserveSource: false
+            type: append
+        - type: string-builder-processor
+          name: Set SHA256 algorithm name
+          enabled: true
+          template: SHA-256
+          target: tmp_sha256.algorithm
+          replaceMissing: false
+        - type: string-builder-processor
+          name: Set SHA256 algorithm id
+          enabled: true
+          template: "3"
+          target: tmp_sha256.algorithm_id
+          replaceMissing: false
+        - type: grok-parser
+          name: Coerce tmp_sha256.algorithm_id to integer
+          enabled: true
+          source: tmp_sha256.algorithm_id
+          samples:
+            - "3"
+          grok:
+            supportRules: ""
+            matchRules: "to_int %{integer:tmp_sha256.algorithm_id}"
+        - type: attribute-remapper
+          name: Map `sha256` to `tmp_sha256.value`
+          enabled: true
+          sources:
+            - sha256
+          sourceType: attribute
+          target: tmp_sha256.value
+          targetType: attribute
+          preserveSource: true
+          overrideOnConflict: false
+        - type: array-processor
+          name: Append tmp_sha256 to ocsf.file.hashes
+          enabled: true
+          operation:
+            source: tmp_sha256
+            target: ocsf.file.hashes
+            preserveSource: false
+            type: append
         - type: schema-processor
           name: Apply OCSF schema for 6006
           enabled: true
@@ -3477,6 +3597,13 @@ pipeline:
               target: ocsf.dst_endpoint.ip
               preserveSource: true
               overrideOnConflict: true
+            - type: schema-remapper
+              name: Map `ocsf.file.hashes` to `ocsf.file.hashes`
+              sources:
+                - ocsf.file.hashes
+              target: ocsf.file.hashes
+              preserveSource: true
+              overrideOnConflict: true
             - type: schema-remapper
               name: Map `mime_type` to `ocsf.file.mime_type`
               sources:

zeek/assets/logs/zeek_tests.yaml

@@ -311,7 +311,7 @@ tests:
       "rejected" : false,
       "query" : "win2k16-1-159",
       "_write_ts" : "2023-12-12T05:52:50.756358Z",
-      "answers" : [ "185.64.148.0" ],
+      "answers" : [ "185.64.148.0", "8.8.8.8" ],
       "trans_id" : 38706,
       "rcode" : 0,
       "_path" : "dns",
@@ -344,7 +344,8 @@ tests:
       dns:
         answer:
           name:
-            - "185.64.148.0"
+            - 185.64.148.0
+            - 8.8.8.8
         flags:
           rcode: "NOERROR"
         id: 38706
@@ -420,6 +421,7 @@ tests:
       query: win2k16-1-159
       answers:
         - 185.64.148.0
+        - 8.8.8.8
       trans_id: 38706
       rcode_name: NOERROR
       proto: udp
@@ -428,15 +430,14 @@ tests:
         resp_h: 185.64.148.0
         orig_h: 185.64.148.0
         resp_p: 5355
-      _answers_str: 185.64.148.0
     message: |-
       {
         "AA" : false,
         "TTLs" : [ 30.0 ],
         "rejected" : false,
         "query" : "win2k16-1-159",
         "_write_ts" : "2023-12-12T05:52:50.756358Z",
-        "answers" : [ "185.64.148.0" ],
+        "answers" : [ "185.64.148.0", "8.8.8.8" ],
         "trans_id" : 38706,
         "rcode" : 0,
         "_path" : "dns",
@@ -994,7 +995,6 @@ tests:
         resp_h: 185.64.148.0
         orig_h: 185.64.148.0
         resp_p: 5355
-      _answers_str: 185.64.148.0
     service: dns
     message: <134>Dec 12 05:52:50 machine-name {"_path":"dns","_write_ts":"2023-12-12T05:52:50.756358Z","ts":"2023-12-12T05:52:32.763303Z","uid":"CsOSdHqRMu62rNs31","id.orig_h":"185.64.148.0","id.orig_p":58013,"id.resp_h":"185.64.148.0","id.resp_p":5355,"proto":"udp","trans_id":38706,"rcode":0,"rcode_name":"NOERROR","query":"win2k16-1-159","answers":["185.64.148.0"],"TTLs":[30.0],"AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}
     tags:
@@ -1274,7 +1274,6 @@ tests:
       severity:
         name: High
         id: 4
-      _is_alert_str: 'true'
       id:
         orig_p: 54321
         resp_h: 192.168.1.1
@@ -1295,7 +1294,6 @@ tests:
       _write_ts: '2026-05-11T17:59:59.359532Z'
       suri_id: SOHaIDWJ5dBe
       _path: suricata_corelight
-      _is_alert_str: 'true'
       tx_id: 0
       network:
         destination:
@@ -1411,6 +1409,16 @@ tests:
           mime_type: text/json
           type_id: 1
           name: FOPDsn3PdkiZsljcj2
+          hashes:
+            - algorithm_id: 1
+              value: 6e6ae0ed19f595687684faafae5499e13
+              algorithm: MD5
+            - algorithm_id: 2
+              value: f6578daa6d398c91398888b91a96d4c0e099c79c
+              algorithm: SHA-1
+            - algorithm_id: 3
+              value: a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f
+              algorithm: SHA-256
         status_id: 1
         class_uid: 6006
         activity_id: 2
@@ -1453,8 +1461,6 @@ tests:
       total_bytes: 253109
       seen_bytes: 253109
       missing_bytes: 0
-      _tx_hosts_str: 10.104.10.60
-      _rx_hosts_str: 10.104.10.65
     service: files
     message: <134>May 11 19:26:26 ndr-dub-stryker-DC-1 {"_path":"files","_system_name":"ndr-dub-stryker-DC-1","_write_ts":"2026-05-11T19:26:26.082433Z","ts":"2026-05-11T19:26:25.875206Z","uid":"CjTuQU17IDvaVa8Nq2","fuid":"FOPDsn3PdkiZsljcj2","tx_hosts":["10.104.10.60"],"rx_hosts":["10.104.10.65"],"conn_uids":["CjTuQU17IDvaGb8Nq2"],"source":"HTTP","depth":0,"analyzers":["SHA1","MD5","SHA256","DATA_EVENT"],"local_orig":true,"is_orig":false,"seen_bytes":253109,"total_bytes":253109,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"duration":0.2072269916534424,"mime_type":"text/json","md5":"6e6ae0ed19f595687684faafae5499e13","sha1":"f6578daa6d398c91398888b91a96d4c0e099c79c","sha256":"a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f","id.vlan":1010}
     tags: