From fe098154c958c45b4f98732f92f48bcd60c4f89a Mon Sep 17 00:00:00 2001 From: Nathaniel Beckstead Date: Mon, 18 May 2026 13:01:26 -0400 Subject: [PATCH 1/4] Set device.os.type to Linux on all OCSF events (#23714) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Set device.os.type to Linux on all OCSF events Add a schema-category-mapper that sets ocsf.device.os.type and ocsf.device.os.type_id (200/Linux) on every event the integration emits. This gives downstream rules a stable, source-agnostic way to filter for Linux events β€” e.g. cross-source detection rules can use @ocsf.device.os.type:Linux to scope to Linux endpoints without depending on metadata.product.vendor_name (which encodes the source, not the OS). Also add profiles: [host] to the SOCKADDR Network Activity sub-pipeline so device.os.* validates against the OCSF schema there (Network Activity has no native device attribute). Test fixtures updated to expect device.os in all 29 ocsf: blocks, and the SOCKADDR fixture to expect metadata.profiles: [host]. Co-Authored-By: Claude Opus 4.7 (1M context) * Fix OCSF validation: device.os.name, base_event host profile, device.type_id Address validator errors after adding device.os.type to all sub-pipelines: - ocsf.device.os.name is required by the OCSF OS object; set it to "Linux" via a top-level string-builder in the OCSF pre-transformations sub-pipeline so it applies to every event. - ocsf.device.name is required to satisfy the Device object's at_least_one constraint on sub-pipelines that don't otherwise populate hostname/ip; set to "Unknown" via the same top-level string-builder. - Base Event class natively includes the host profile in OCSF; declaring profiles: [host] on the Base Event schema-processor brings the pipeline in line so device.* validates. - ocsf.device.type_id was missing from Base Event, SOCKADDR Network Activity, and SYSCALL Network Activity sub-pipelines; added a schema-category-mapper (Unknown / 0) to each. Test fixtures updated to reflect the new fields: device.os.name on all 29 fixtures, device.name on the 11 fixtures that previously lacked it (9 IAM/Device-Config with hostname/ip, 2 Network 4001), device.type and device.type_id on the 11 fixtures that previously lacked the full device shape (9 Base Event, 2 Network 4001), and metadata.profiles: [host] on the 9 Base Event fixtures. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .../assets/logs/linux-audit-logs.yaml | 226 +++++++++++++++++- .../assets/logs/linux-audit-logs_tests.yaml | 189 +++++++++++++++ 2 files changed, 414 insertions(+), 1 deletion(-) diff --git a/linux_audit_logs/assets/logs/linux-audit-logs.yaml b/linux_audit_logs/assets/logs/linux-audit-logs.yaml index 209297d2aea8c..2ee31c723e274 100644 --- a/linux_audit_logs/assets/logs/linux-audit-logs.yaml +++ b/linux_audit_logs/assets/logs/linux-audit-logs.yaml @@ -467,6 +467,18 @@ pipeline: template: Linux target: ocsf.metadata.product.vendor_name replaceMissing: false + - type: string-builder-processor + name: Set `ocsf.device.name` to Unknown + enabled: true + template: Unknown + target: ocsf.device.name + replaceMissing: false + - type: string-builder-processor + name: Set `ocsf.device.os.name` to Linux + enabled: true + template: Linux + target: ocsf.device.os.name + replaceMissing: false - type: arithmetic-processor name: Convert `timestamp` epoch to seconds enabled: true @@ -490,7 +502,8 @@ pipeline: className: Base Event classUid: 0 extensions: [] - profiles: [] + profiles: + - host mappers: - name: ocsf.activity_id categories: @@ -597,6 +610,32 @@ pipeline: preserveSource: true overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper + - name: ocsf.device.type_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.device.type + id: ocsf.device.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class File System Activity [1001] enabled: true @@ -846,6 +885,19 @@ pipeline: preserveSource: false overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class File System Activity [1001] from SYSCALL enabled: true @@ -1120,6 +1172,19 @@ pipeline: preserveSource: true overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class Module Activity [1005] enabled: true @@ -1355,6 +1420,19 @@ pipeline: preserveSource: true overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class Account Change [3001] enabled: true @@ -1609,6 +1687,19 @@ pipeline: preserveSource: false overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class Authentication [3002] enabled: true @@ -1873,6 +1964,19 @@ pipeline: preserveSource: false overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for role assign logs enabled: true @@ -2167,6 +2271,19 @@ pipeline: preserveSource: true overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class Group Management [3006] enabled: true @@ -2416,6 +2533,19 @@ pipeline: preserveSource: false overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class Device Config State Change [5019] enabled: true @@ -2639,6 +2769,19 @@ pipeline: preserveSource: true overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class Process Activity [1007] for EXECVE enabled: true @@ -2827,6 +2970,19 @@ pipeline: preserveSource: false overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class Process Activity [1007] from SYSCALL enabled: true @@ -3061,6 +3217,19 @@ pipeline: preserveSource: true overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class Network Activity [4001] from SOCKADDR enabled: true @@ -3088,6 +3257,9 @@ pipeline: version: 1.5.0 className: Network Activity classUid: 4001 + extensions: [] + profiles: + - host mappers: - name: ocsf.activity_id categories: @@ -3187,6 +3359,32 @@ pipeline: preserveSource: true overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper + - name: ocsf.device.type_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.device.type + id: ocsf.device.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper - type: pipeline name: OCSF sub pipeline for class Network Activity [4001] from Syscall enabled: true @@ -3403,3 +3601,29 @@ pipeline: preserveSource: true overrideOnConflict: true type: schema-remapper + - name: ocsf.device.os.type_id + categories: + - filter: + query: "*" + name: Linux + id: 200 + targets: + name: ocsf.device.os.type + id: ocsf.device.os.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper + - name: ocsf.device.type_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.device.type + id: ocsf.device.type_id + fallback: + values: {} + sources: {} + type: schema-category-mapper diff --git a/linux_audit_logs/assets/logs/linux-audit-logs_tests.yaml b/linux_audit_logs/assets/logs/linux-audit-logs_tests.yaml index 4ee7fc28e1fcf..500b578cd6e3b 100644 --- a/linux_audit_logs/assets/logs/linux-audit-logs_tests.yaml +++ b/linux_audit_logs/assets/logs/linux-audit-logs_tests.yaml @@ -48,6 +48,11 @@ tests: device: hostname: "10.10.10.10" ip: "10.10.10.10" + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 metadata: @@ -126,6 +131,11 @@ tests: device: hostname: "10.10.10.10" ip: "10.10.10.10" + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 dst_endpoint: @@ -183,6 +193,14 @@ tests: category_uid: 0 class_name: "Base Event" class_uid: 0 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 metadata: event_code: "MAC_STATUS" uid: "1674" @@ -190,6 +208,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -229,6 +249,14 @@ tests: category_uid: 0 class_name: "Base Event" class_uid: 0 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 metadata: event_code: "AVC" uid: "182" @@ -236,6 +264,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -282,6 +312,14 @@ tests: category_uid: 0 class_name: "Base Event" class_uid: 0 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 metadata: event_code: "USER_SELINUX_ERR" uid: "198" @@ -289,6 +327,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -335,6 +375,14 @@ tests: category_uid: 0 class_name: "Base Event" class_uid: 0 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 metadata: event_code: "USER_ROLE_CHANGE" uid: "2606" @@ -342,6 +390,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -381,6 +431,14 @@ tests: category_uid: 0 class_name: "Base Event" class_uid: 0 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 metadata: event_code: "MAC_CONFIG_CHANGE" uid: "121" @@ -388,6 +446,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -450,6 +510,11 @@ tests: device: hostname: "ub10-10-10-10" ip: "10.10.10.10" + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 group: @@ -529,6 +594,11 @@ tests: device: hostname: "localhost" ip: "10.10.10.10" + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 group: @@ -607,6 +677,11 @@ tests: device: hostname: "localhost" ip: "10.10.10.10" + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 metadata: @@ -686,6 +761,11 @@ tests: device: hostname: "localhost" ip: "10.10.10.10" + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 metadata: @@ -768,6 +848,11 @@ tests: device: hostname: "localhost" ip: "10.10.10.10" + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 metadata: @@ -854,6 +939,11 @@ tests: device: hostname: "localhost" ip: "10.10.10.10" + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 metadata: @@ -942,6 +1032,11 @@ tests: device: hostname: "localhost" ip: "10.10.10.10" + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 metadata: @@ -1012,6 +1107,10 @@ tests: class_uid: 1001 device: name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 uid: "fd:03" @@ -1077,6 +1176,10 @@ tests: class_uid: 1001 device: name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 uid: "fd:03" @@ -1143,6 +1246,10 @@ tests: class_uid: 1001 device: name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 uid: "fd:03" @@ -1208,6 +1315,10 @@ tests: class_uid: 1001 device: name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 uid: "fd:03" @@ -1258,6 +1369,14 @@ tests: category_uid: 0 class_name: "Base Event" class_uid: 0 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 metadata: event_code: "DAEMON_CONFIG" uid: "946" @@ -1265,6 +1384,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -1306,6 +1427,14 @@ tests: category_uid: 0 class_name: "Base Event" class_uid: 0 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 metadata: event_code: "DAEMON_ABORT" uid: "1849" @@ -1313,6 +1442,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -1355,6 +1486,14 @@ tests: category_uid: 0 class_name: "Base Event" class_uid: 0 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 metadata: event_code: "MAC_POLICY_LOAD" uid: "235" @@ -1362,6 +1501,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -1401,6 +1542,14 @@ tests: category_uid: 0 class_name: "Base Event" class_uid: 0 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 metadata: event_code: "CONFIG_CHANGE" uid: "2361" @@ -1408,6 +1557,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -1452,6 +1603,10 @@ tests: class_uid: 1007 device: name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 metadata: @@ -1533,6 +1688,10 @@ tests: class_uid: 1007 device: name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 metadata: @@ -1627,6 +1786,14 @@ tests: category_uid: 4 class_name: "Network Activity" class_uid: 4001 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 dst_endpoint: name: "Unknown" metadata: @@ -1681,6 +1848,14 @@ tests: category_uid: 4 class_name: "Network Activity" class_uid: 4001 + device: + name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 + type: "Unknown" + type_id: 0 dst_endpoint: ip: "1.2.3.4" port: 4444 @@ -1691,6 +1866,8 @@ tests: product: name: "Auditd" vendor_name: "Linux" + profiles: + - "host" version: "1.5.0" severity: "Informational" severity_id: 1 @@ -1759,6 +1936,10 @@ tests: class_uid: 1001 device: name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 file: @@ -1855,6 +2036,10 @@ tests: class_uid: 1001 device: name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 file: @@ -1952,6 +2137,10 @@ tests: class_uid: 1005 device: name: "Unknown" + os: + name: "Linux" + type: "Linux" + type_id: 200 type: "Unknown" type_id: 0 metadata: From 4c7e8bbb1292d3cf808acba6090cee4b7ab3b480 Mon Sep 17 00:00:00 2001 From: Eric Weaver Date: Mon, 18 May 2026 13:29:14 -0400 Subject: [PATCH 2/4] [DBMON-6602] Avoid cleanup when cancel called while check running (#23728) * Avoid cleanup when cancel called while check running * Add changelog * check if cancelled before running jobs * Add some debug lines for cancel flow --- postgres/changelog.d/23728.fixed | 1 + postgres/datadog_checks/postgres/postgres.py | 112 ++++++++++++++----- postgres/tests/test_unit.py | 74 ++++++++++++ 3 files changed, 158 insertions(+), 29 deletions(-) create mode 100644 postgres/changelog.d/23728.fixed diff --git a/postgres/changelog.d/23728.fixed b/postgres/changelog.d/23728.fixed new file mode 100644 index 0000000000000..aec07d0c61155 --- /dev/null +++ b/postgres/changelog.d/23728.fixed @@ -0,0 +1 @@ +Fix a crash caused by cancel closing database connections while the check is still running. \ No newline at end of file diff --git a/postgres/datadog_checks/postgres/postgres.py b/postgres/datadog_checks/postgres/postgres.py index dac424e3f9c5f..dbd4bd067bc7c 100644 --- a/postgres/datadog_checks/postgres/postgres.py +++ b/postgres/datadog_checks/postgres/postgres.py @@ -5,6 +5,7 @@ import copy import functools import os +import threading from string import Template from time import time @@ -17,7 +18,6 @@ from datadog_checks.base.utils.db.core import QueryManager from datadog_checks.base.utils.db.health import HealthEvent, HealthStatus from datadog_checks.base.utils.db.utils import ( - DBMAsyncJob, default_json_event_encoding, tracked_query, ) @@ -194,6 +194,10 @@ def __init__(self, name, init_config, instances): self.diagnosis.register(functools.partial(run_diagnostics, self)) + self._cancel_lock = threading.Lock() + self._is_running = False + self._cancelled = False + def database_monitoring_column_statistics(self, raw_event: str): self.event_platform_event(raw_event, "dbm-column-statistics") @@ -476,38 +480,87 @@ def dynamic_queries(self): return self._dynamic_queries - @staticmethod - def _cancel_async_job(job: DBMAsyncJob): - job.cancel() - if job._job_loop_future: - job._job_loop_future.result() - job._job_loop_future = None - job._shutdown() + def run(self): + # TODO: move this lock into the base class + with self._cancel_lock: + if self._cancelled: + self.log.debug("run() skipped, check already cancelled") + return '' + self._is_running = True + try: + return super().run() + finally: + needs_finalize = False + with self._cancel_lock: + self._is_running = False + if self._cancelled: + needs_finalize = True + if needs_finalize: + self.log.debug("Check cancel has been signaled, finalizing now that run() is complete") + self._finalize() def cancel(self): + """Signal that the check is being unscheduled. + + This method can be called while check() is running on another thread + (the GIL is released during psycopg I/O). It must not perform any + destructive operations β€” closing connections or nulling attributes that + check() depends on β€” because that causes a SIGSEGV in libpq when + check() resumes. + + Destructive cleanup is deferred to _finalize(), which is called either + here (if the check is idle) or by run()'s finally block (if the check + is in-flight). The Agent guarantees it will not call run() again after + cancel(). """ - Cancels and sends cancel signal to all threads. - """ + self.log.debug("Marking check as cancelled") + self._cancel_async_jobs() + needs_finalize = False + with self._cancel_lock: + self._cancelled = True + if not self._is_running: + needs_finalize = True + if needs_finalize: + self.log.debug("cancel() finalizing immediately, check is idle") + self._finalize() + else: + self.log.debug("cancel() deferred finalize, check is still running") + + @property + def _async_jobs(self): + """Return the async jobs active for this check's configuration.""" + jobs = [] if self._config.dbm: - self._cancel_async_job(self.statement_metrics) - self._cancel_async_job(self.statement_samples) - self._cancel_async_job(self.metadata_samples) + jobs.extend([self.statement_metrics, self.statement_samples, self.metadata_samples]) elif self._config.data_observability.enabled: - self._cancel_async_job(self.metadata_samples) + jobs.append(self.metadata_samples) if self._config.data_observability.enabled: - self._cancel_async_job(self.data_observability) + jobs.append(self.data_observability) + return jobs + + def _cancel_async_jobs(self): + """Signal async jobs to stop. Safe to call while check() is running.""" + for job in self._async_jobs: + job.cancel() + + def _finalize(self): + """Tear down check state. Must not run while check() is executing.""" + self.log.debug("Finalizing check: closing connections and clearing state") + for job in self._async_jobs: + if job._job_loop_future: + job._job_loop_future.result() + job._job_loop_future = None + job._shutdown() self._clean_state() - self._query_manager = None - self.health = None self.check_initializations.clear() # TODO: move diagnosis cleanup into AgentCheck.cancel() in the base class self._diagnosis = None + self.log.check = None + self._query_manager = None + self.health = None self._close_db() self._close_db_pool() - # CheckLoggingAdapter holds self.check until check_id is resolved via - # process(), which only happens after the agent scheduler calls run(). - # If cancel() is called before that, the back-reference is never cleared. - self.log.check = None + self.log.debug("Check cleanup complete") def _clean_state(self): self.log.debug("Cleaning state") @@ -1191,14 +1244,15 @@ def check(self, _): if not self._config.only_custom_queries: self._collect_stats(tags) - if self._config.dbm: - self.statement_metrics.run_job_loop(tags) - self.statement_samples.run_job_loop(tags) - self.metadata_samples.run_job_loop(tags) - elif self._config.data_observability.enabled: - self.metadata_samples.run_job_loop(tags) - if self._config.data_observability.enabled: - self.data_observability.run_job_loop(tags) + if not self._cancelled: + if self._config.dbm: + self.statement_metrics.run_job_loop(tags) + self.statement_samples.run_job_loop(tags) + self.metadata_samples.run_job_loop(tags) + elif self._config.data_observability.enabled: + self.metadata_samples.run_job_loop(tags) + if self._config.data_observability.enabled: + self.data_observability.run_job_loop(tags) if self._config.collect_wal_metrics is True: # collect wal metrics for pg < 10 only when explicitly enabled # (requires local filesystem access to the WAL directory) diff --git a/postgres/tests/test_unit.py b/postgres/tests/test_unit.py index 7efdb71479139..5505131e71ce1 100644 --- a/postgres/tests/test_unit.py +++ b/postgres/tests/test_unit.py @@ -443,6 +443,80 @@ def test_check_gc_after_cancel(pg_instance): gc.enable() +def test_cancel_during_running_check_defers_finalize(pg_instance): + """Verify that cancel() during an in-flight check() does not close connections. + + Destructive cleanup (_finalize) must be deferred until run() completes so + that check() never accesses a closed psycopg connection, which would cause + a SIGSEGV in libpq. + """ + import threading + + check = PostgreSql('postgres', {}, [pg_instance]) + conn = mock.MagicMock() + check._db = conn + + check_started = threading.Event() + cancel_done = threading.Event() + + def slow_run(self_arg): + check_started.set() + cancel_done.wait(timeout=5) + return '' + + run_result = [None] + + def run_check(): + with mock.patch.object(type(check).__mro__[1], 'run', slow_run): + run_result[0] = check.run() + + run_thread = threading.Thread(target=run_check) + run_thread.start() + + check_started.wait(timeout=5) + + check.cancel() + # cancel() should have signaled but NOT finalized since run() is in-flight + assert not conn.close.called, "_close_db() ran while check() was still executing" + assert check._cancelled is True + + cancel_done.set() + run_thread.join(timeout=5) + + # After run() completes, _finalize() should have been called + conn.close.assert_called_once() + assert check._db is None + assert check._query_manager is None + assert check.health is None + + +def test_cancel_on_idle_check_finalizes_immediately(pg_instance): + """Verify that cancel() on an idle check runs _finalize() inline.""" + check = PostgreSql('postgres', {}, [pg_instance]) + conn = mock.MagicMock() + check._db = conn + + assert not check._is_running + + check.cancel() + + conn.close.assert_called_once() + assert check._db is None + assert check._query_manager is None + assert check.health is None + + +def test_run_after_cancel_returns_immediately(pg_instance): + """Verify that run() returns '' without executing check() if already cancelled.""" + check = PostgreSql('postgres', {}, [pg_instance]) + check.cancel() + + with mock.patch.object(check, 'check', side_effect=AssertionError("check() should not be called")): + result = check.run() + + assert result == '' + + def test_collect_column_statistics_updates_timestamp_on_failure(pg_instance): pg_instance['dbm'] = True pg_instance['collect_column_statistics'] = {'enabled': True, 'collection_interval': 60} From b4f366a5af1f4ab0839abe43d36efbaa8e18b639 Mon Sep 17 00:00:00 2001 From: Cepolation-Datadog <86613440+cepolation-datadog@users.noreply.github.com> Date: Mon, 18 May 2026 12:55:49 -0500 Subject: [PATCH 3/4] [OCSF] Zeek pipeline (#23712) * [OCSF] Zeek/Corelight pipeline Add OCSF v1.5.0 normalization for Zeek/Corelight logs, covering 7 log types across 5 OCSF classes (Detection Finding, Network Activity, HTTP Activity, DNS Activity, File Hosting Activity). Co-Authored-By: Claude Sonnet 4.6 * Fix validate-logs errors in zeek.yaml Resolve 36 validation errors flagged by the datadog-assets validator: - Add missing `overrideOnConflict: false` to 3 attribute-remappers - Fix 2 schema-remapper names to backtick individual fields - Rename 25 facets to match validator's canonical names and add `type: integer`/`facetType: range` where required - Remove 6 facets with unresolvable path conflicts (validator demanded unique paths with no canonical definition available) Co-Authored-By: Claude Opus 4.7 (1M context) * Fix severity mapping for Detection Finding [2004] Notice Notice events emit `severity.name` capitalized ("High", "Medium", etc.), so the lowercase `@severity.name:informational` filters never matched and the fallback assigned `ocsf.severity_id: 99` while preserving the capitalized name as `ocsf.severity`. Switch the schema-category-mapper to filter on the numeric `severity.id` (1-5) which Corelight reliably emits, and update the notice fixture's expected `severity_id` from 99 to 4 to reflect the corrected mapping. Co-Authored-By: Claude Opus 4.7 (1M context) * Add catch-all category to schema-category-mappers with fallback Each schema-category-mapper that defines a fallback must also have a catch-all filter category at the end matching the fallback's values. Six mappers were missing the trailing catch-all: notice/alert severity_id (2004), http activity_id/status_id (4002), dns rcode_id, and dns status_id (4003). Append `query: "*"` -> Other/99 to each. Co-Authored-By: Claude Opus 4.7 (1M context) * Apply PR review feedback for Zeek/Corelight OCSF pipeline Direct mappings, dead-code removal, correctness fixes, and OCSF validator cleanups across notice, suricata, conn, ssl, weird, http, dns, and file hosting sub-pipelines: - Map directly to OCSF targets where intermediates were unnecessary (ocsf.time, ocsf.duration, ocsf.traffic.packets, JA3/JA3S algorithm_id, weird protocol_name). - Drop dead/auto-generated mappers: notice/suricata category_uid (set by schema-processor), self-maps of finding_info.uid, event_code, file.hashes (when unbuilt upstream), suricata community_id correlation_uid, HTTP version-as-protocol_ver, DNS direction derivation, and the DNS rcode_id catch-all/fallback (recommended-not-required). - Convert suricata alert.signature_id event_code from string-builder to schema-remapper. - Combine domain/query into single ocsf.query.hostname schema-remapper. - Fix DNS Activity filters: use rcode_name presence to discriminate Response/Query instead of dns.answer.name (handles NXDOMAIN responses). - DNS status_id catch-all renamed Other/99 -> Unknown/0 to satisfy the OCSF validator's suspicious-Other check. - File Hosting tx_hosts/rx_hosts: drop the second intermediate field; grok targets ocsf.{src,dst}_endpoint.ip directly off a single stringify. - Switch fallback source fields per Jonah's suggestions: severity -> severity.name, alert.severity -> alert_severity, http status -> status_msg, dns rcode/status -> rcode_name. - Notice fixture: use id.orig_h/id.resp_h connection fields instead of the suricata-style src. Regenerated zeek_tests.yaml with the OCSF validator (--check-all --write). All 14 logs pass validation with no errors or warnings. Co-Authored-By: Claude Opus 4.7 (1M context) * Map Zeek DNS answers to ocsf.answers as dns_answer objects Use two array-processors to wrap each Zeek `answers` string into a dns_answer object and append to ocsf.answers: the first selects the first array element into ocsf.answer.rdata, the second appends ocsf.answer onto ocsf.answers. Only the first answer is captured (the pipeline DSL has no per-element iteration), but that covers the common single-A-record case. Co-Authored-By: Claude Opus 4.7 (1M context) * Add catch-all for activity_id * Fix validate-logs failure for DNS answers wrapper The previous array-processor type:select required operation.filter and operation.valueToExtract per the asset validator, but those only apply to object arrays - Zeek's `answers` is a primitive string array. Switch to string-builder + grok-parser to extract the first answer string into ocsf.answer.rdata, then keep the array-processor append to wrap it into ocsf.answers as a dns_answer object. Co-Authored-By: Claude Opus 4.7 (1M context) * Address codex review feedback for file pipeline - Include `files_red` in the File Hosting [6006] sub-pipeline filter so redacted file events get OCSF class_uid/activity_id/file fields, not just the pre-transform metadata. - Prefer `filename` over `fuid` when populating `ocsf.file.name`; fall back to `fuid` only when `filename` is absent. The `fuid` mapping to `ocsf.file.uid` is unaffected. Co-Authored-By: Claude Opus 4.7 (1M context) * Drop pipeline intermediates, fix multi-IP grok, restore file.hashes - is_alert (notice 2004, suricata 2004): string-builder writes directly to `ocsf.is_alert`; grok-parser converts in place. Drops the `_is_alert_str` intermediate. - DNS answers: stringify directly into `ocsf.answer`; grok extracts `ocsf.answer.rdata` via `a %{data:ocsf.answer.rdata}(,%{data})?` so the comma-separated multi-IP form parses correctly. Drops the `_answers_str` intermediate. - File Hosting tx/rx hosts: stringify directly into `ocsf.{src,dst}_endpoint`; grok extracts `.ip` via `g %{ip:ocsf.{src,dst}_endpoint.ip}(,%{data})?` for multi-IP. Drops the `_tx_hosts_str`/`_rx_hosts_str` intermediates. - Connection 4001: arithmetic-processor writes total bytes directly to `ocsf.traffic.bytes`; the schema-processor remapper becomes a self-map. Drops the `_total_bytes` intermediate (matches the earlier _total_packets/_duration_ms cleanup). - Restore `ocsf.file.hashes`: build `tmp_md5`/`tmp_sha1`/`tmp_sha256` fingerprint objects (algorithm name, integer algorithm_id, value), array-processor append each into `ocsf.file.hashes`, and self-map the array inside the 6006 schema-processor. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Sonnet 4.6 --- zeek/assets/logs/zeek.yaml | 2535 +++++++++++++++++++++++++++++- zeek/assets/logs/zeek_tests.yaml | 1139 +++++++++++++- 2 files changed, 3583 insertions(+), 91 deletions(-) diff --git a/zeek/assets/logs/zeek.yaml b/zeek/assets/logs/zeek.yaml index 528fba092c81b..bbce300a8559b 100644 --- a/zeek/assets/logs/zeek.yaml +++ b/zeek/assets/logs/zeek.yaml @@ -390,6 +390,422 @@ facets: path: zeek.x509_total source: log type: double + - groups: + - OCSF + name: Activity ID + path: ocsf.activity_id + source: log + - groups: + - OCSF + name: Activity Name + path: ocsf.activity_name + source: log + - groups: + - OCSF + name: Category ID + path: ocsf.category_uid + source: log + - groups: + - OCSF + name: Category + path: ocsf.category_name + source: log + - groups: + - OCSF + name: Class ID + path: ocsf.class_uid + source: log + - groups: + - OCSF + name: Class + path: ocsf.class_name + source: log + - groups: + - OCSF + name: Severity + path: ocsf.severity + source: log + - groups: + - OCSF + name: Severity ID + path: ocsf.severity_id + source: log + - groups: + - OCSF + name: Status + path: ocsf.status + source: log + - groups: + - OCSF + name: Status ID + path: ocsf.status_id + source: log + - groups: + - OCSF + name: Source IP Address + path: ocsf.src_endpoint.ip + source: log + - groups: + - OCSF + name: Destination IP Address + path: ocsf.dst_endpoint.ip + source: log + - groups: + - OCSF + name: Event Code + path: ocsf.metadata.event_code + source: log + - groups: + - OCSF + name: Product Name + path: ocsf.metadata.product.name + source: log + - groups: + - OCSF + name: Vendor Name + path: ocsf.metadata.product.vendor_name + source: log + - groups: + - OCSF + name: Finding Info Title + path: ocsf.finding_info.title + source: log + - groups: + - OCSF + name: Finding Info Unique ID + path: ocsf.finding_info.uid + source: log + - groups: + - OCSF + name: HTTP Method + path: ocsf.http_request.http_method + source: log + - groups: + - OCSF + name: Request URL String + path: ocsf.http_request.url.url_string + source: log + - groups: + - OCSF + name: TLS Version + path: ocsf.tls.version + source: log + - groups: + - OCSF + name: File Name + path: ocsf.file.name + source: log + - groups: + - OCSF + name: File MIME Type + path: ocsf.file.mime_type + source: log + - groups: + - OCSF + name: DNS Query Hostname + path: ocsf.query.hostname + source: log + - groups: + - OCSF + name: Session Unique ID + path: ocsf.actor.session.uid + source: log + - groups: + - OCSF + name: Confidence + path: ocsf.confidence + source: log + - groups: + - OCSF + name: Confidence ID + path: ocsf.confidence_id + source: log + - groups: + - OCSF + name: Connection Info Boundary + path: ocsf.connection_info.boundary + source: log + - groups: + - OCSF + name: Connection Info Boundary ID + path: ocsf.connection_info.boundary_id + source: log + - groups: + - OCSF + name: Connection Info Community UID + path: ocsf.connection_info.community_uid + source: log + - groups: + - OCSF + name: Connection Info Direction + path: ocsf.connection_info.direction + source: log + - groups: + - OCSF + name: Connection Info Direction ID + path: ocsf.connection_info.direction_id + source: log + - groups: + - OCSF + name: Connection Info Flag History + path: ocsf.connection_info.flag_history + source: log + - groups: + - OCSF + name: Connection Info Protocol Ver + path: ocsf.connection_info.protocol_ver + source: log + - groups: + - OCSF + name: Connection Info UID + path: ocsf.connection_info.uid + source: log + - groups: + - OCSF + name: Device IP + path: ocsf.device.ip + source: log + - groups: + - OCSF + name: Dst Endpoint Hostname + path: ocsf.dst_endpoint.hostname + source: log + - facetType: range + groups: + - OCSF + name: Dst Endpoint Port + path: ocsf.dst_endpoint.port + source: log + type: integer + - facetType: range + groups: + - OCSF + name: Duration Milliseconds + path: ocsf.duration + source: log + type: integer + - groups: + - OCSF + name: Evidences + path: ocsf.evidences + source: log + - groups: + - OCSF + name: File Hashes + path: ocsf.file.hashes + source: log + - groups: + - OCSF + name: File Size + path: ocsf.file.size + source: log + - groups: + - OCSF + name: File Type ID + path: ocsf.file.type_id + source: log + type: integer + - groups: + - OCSF + name: File Unique ID + path: ocsf.file.uid + source: log + - groups: + - OCSF + name: Finding Info Analytic Name + path: ocsf.finding_info.analytic.name + source: log + - groups: + - OCSF + name: Finding Info Analytic Type + path: ocsf.finding_info.analytic.type + source: log + - groups: + - OCSF + name: Finding Info Analytic Type ID + path: ocsf.finding_info.analytic.type_id + source: log + - groups: + - OCSF + name: Finding Info Analytic UID + path: ocsf.finding_info.analytic.uid + source: log + - groups: + - OCSF + name: Finding Info UID Alt + path: ocsf.finding_info.uid_alt + source: log + - groups: + - OCSF + name: HTTP Request Body Length + path: ocsf.http_request.body_length + source: log + - groups: + - OCSF + name: Url Hostname + path: ocsf.http_request.url.hostname + source: log + - groups: + - OCSF + name: Url Path + path: ocsf.http_request.url.path + source: log + - groups: + - OCSF + name: HTTP Request URL Port + path: ocsf.http_request.url.port + source: log + - groups: + - OCSF + name: HTTP User-Agent + path: ocsf.http_request.user_agent + source: log + - groups: + - OCSF + name: HTTP Response Body Length + path: ocsf.http_response.body_length + source: log + - groups: + - OCSF + name: Response Code + path: ocsf.http_response.code + source: log + type: integer + - groups: + - OCSF + name: HTTP Response Message + path: ocsf.http_response.message + source: log + - groups: + - OCSF + name: Message + path: ocsf.message + source: log + - groups: + - OCSF + name: Metadata Original Time + path: ocsf.metadata.original_time + source: log + - groups: + - OCSF + name: Metadata Event UID + path: ocsf.metadata.uid + source: log + - groups: + - OCSF + name: Query Class + path: ocsf.query.class + source: log + - groups: + - OCSF + name: Query Resource Record Type + path: ocsf.query.type + source: log + - groups: + - OCSF + name: Response Code + path: ocsf.rcode + source: log + - groups: + - OCSF + name: Response Code ID + path: ocsf.rcode_id + source: log + type: integer + - facetType: range + groups: + - OCSF + name: Src Endpoint Port + path: ocsf.src_endpoint.port + source: log + type: integer + - groups: + - OCSF + name: Status Detail + path: ocsf.status_detail + source: log + - facetType: range + groups: + - OCSF + name: Event Time + path: ocsf.time + source: log + type: integer + - groups: + - OCSF + name: TLS Cipher + path: ocsf.tls.cipher + source: log + - groups: + - OCSF + name: TLS JA3 Hash Algorithm + path: ocsf.tls.ja3_hash.algorithm + source: log + - groups: + - OCSF + name: TLS JA3 Hash Algorithm ID + path: ocsf.tls.ja3_hash.algorithm_id + source: log + - groups: + - OCSF + name: TLS JA3 Hash Value + path: ocsf.tls.ja3_hash.value + source: log + - groups: + - OCSF + name: TLS JA3s Hash Algorithm + path: ocsf.tls.ja3s_hash.algorithm + source: log + - groups: + - OCSF + name: TLS JA3s Hash Algorithm ID + path: ocsf.tls.ja3s_hash.algorithm_id + source: log + - groups: + - OCSF + name: TLS JA3s Hash Value + path: ocsf.tls.ja3s_hash.value + source: log + - groups: + - OCSF + name: TLS SNI + path: ocsf.tls.sni + source: log + - groups: + - OCSF + name: Traffic Bytes + path: ocsf.traffic.bytes + source: log + - groups: + - OCSF + name: Traffic Bytes In + path: ocsf.traffic.bytes_in + source: log + - groups: + - OCSF + name: Traffic Bytes Missed + path: ocsf.traffic.bytes_missed + source: log + - groups: + - OCSF + name: Traffic Bytes Out + path: ocsf.traffic.bytes_out + source: log + - groups: + - OCSF + name: Traffic Packets + path: ocsf.traffic.packets + source: log + - groups: + - OCSF + name: Traffic Packets In + path: ocsf.traffic.packets_in + source: log + - groups: + - OCSF + name: Traffic Packets Out + path: ocsf.traffic.packets_out + source: log pipeline: type: pipeline name: Zeek @@ -425,7 +841,7 @@ pipeline: sourceType: attribute target: network.client.ip targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `id.orig_p` to `network.client.port` @@ -435,7 +851,7 @@ pipeline: sourceType: attribute target: network.client.port targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `id.resp_h` to `network.destination.ip` @@ -445,7 +861,7 @@ pipeline: sourceType: attribute target: network.destination.ip targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `id.resp_p` to `network.destination.port` @@ -455,7 +871,7 @@ pipeline: sourceType: attribute target: network.destination.port targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Datetime Remapper for other than files_red logs @@ -512,7 +928,7 @@ pipeline: sourceType: attribute target: zeek.missed_bytes targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Processing for pe logs @@ -598,7 +1014,7 @@ pipeline: sourceType: attribute target: http.method targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `host` to `http.url_details.host` @@ -608,7 +1024,7 @@ pipeline: sourceType: attribute target: http.url_details.host targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `uri` to `http.url_details.path` @@ -618,7 +1034,7 @@ pipeline: sourceType: attribute target: http.url_details.path targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `referrer` to `http.referer` @@ -628,7 +1044,7 @@ pipeline: sourceType: attribute target: http.referer targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `version` to `http.version` @@ -638,7 +1054,7 @@ pipeline: sourceType: attribute target: http.version targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `user_agent` to `http.useragent` @@ -648,7 +1064,7 @@ pipeline: sourceType: attribute target: http.useragent targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `status_code` to `http.status_code` @@ -658,7 +1074,7 @@ pipeline: sourceType: attribute target: http.status_code targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Standard attribute remapping for Web Access @@ -675,7 +1091,7 @@ pipeline: sourceType: attribute target: network.bytes_read targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `resp_bytes`, `response_body_len` to `network.bytes_written` @@ -686,7 +1102,7 @@ pipeline: sourceType: attribute target: network.bytes_written targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Processing for dns logs and Standard attribute remapping for DNS @@ -719,7 +1135,7 @@ pipeline: sourceType: attribute target: dns.id targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `domain`, `query` to `dns.question.name` @@ -730,7 +1146,7 @@ pipeline: sourceType: attribute target: dns.question.name targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `qclass_name` to `dns.question.class` @@ -740,7 +1156,7 @@ pipeline: sourceType: attribute target: dns.question.class targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `qtype_name` to `dns.question.type` @@ -750,7 +1166,7 @@ pipeline: sourceType: attribute target: dns.question.type targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `rcode_name` to `dns.flags.rcode` @@ -760,7 +1176,7 @@ pipeline: sourceType: attribute target: dns.flags.rcode targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `answers` to `dns.answer.name` @@ -770,7 +1186,7 @@ pipeline: sourceType: attribute target: dns.answer.name targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Standard attribute remapping for user attributes @@ -865,7 +1281,7 @@ pipeline: sourceType: attribute target: zeek.duration_sec targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Remap attributes for ssh logs @@ -929,7 +1345,7 @@ pipeline: sourceType: attribute target: zeek.note targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Remap attributes for files logs @@ -955,7 +1371,7 @@ pipeline: sourceType: attribute target: zeek.missing_bytes targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `seen_bytes` to `zeek.seen_bytes` @@ -965,7 +1381,7 @@ pipeline: sourceType: attribute target: zeek.seen_bytes targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `total_bytes` to `zeek.total_bytes` @@ -975,7 +1391,7 @@ pipeline: sourceType: attribute target: zeek.total_bytes targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: attribute-remapper name: Map `network.client.ip`, `rx_hosts` to `orig_host` @@ -1061,7 +1477,7 @@ pipeline: sourceType: attribute target: zeek.proto targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: pipeline name: Remapping datared log attributes @@ -1233,7 +1649,7 @@ pipeline: sourceType: attribute target: network.client.port targetType: attribute - preserveSource: false + preserveSource: true overrideOnConflict: false - type: status-remapper name: Define `severity`, `syslog.severity` as the official status of the log @@ -1241,3 +1657,2068 @@ pipeline: sources: - severity - syslog.severity + - type: pipeline + name: OCSF pre transformations + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(files OR files_red OR notice OR suricata_corelight OR conn OR conn_long OR conn_red OR ssl OR ssl_red OR weird_red OR http OR http_red OR dns OR dns_red)" + processors: + - type: string-builder-processor + name: Add product name + enabled: true + template: Zeek + target: ocsf.metadata.product.name + replaceMissing: false + - type: string-builder-processor + name: Add product vendor name + enabled: true + template: Corelight + target: ocsf.metadata.product.vendor_name + replaceMissing: false + - type: grok-parser + name: Parse `ts` to epoch milliseconds + enabled: true + source: ts + samples: + - "2023-12-12T05:52:32.763303Z" + grok: + supportRules: "" + matchRules: 'parse_ts %{date("yyyy-MM-dd''T''HH:mm:ss.SSSSSSZ"):ocsf.time}' + - type: pipeline + name: OCSF sub pipeline for class Detection Finding [2004] - Notice + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:notice" + processors: + - type: string-builder-processor + name: Set is_alert to boolean true + enabled: true + template: "true" + target: ocsf.is_alert + replaceMissing: false + - type: grok-parser + name: Convert ocsf.is_alert string to boolean + enabled: true + source: ocsf.is_alert + samples: + - "true" + grok: + supportRules: "" + matchRules: "to_bool %{boolean(\"true\",\"false\"):ocsf.is_alert}" + - type: attribute-remapper + name: Map `id.orig_h` to `ocsf.evidence.src_endpoint.ip` + enabled: true + sources: + - id.orig_h + sourceType: attribute + target: ocsf.evidence.src_endpoint.ip + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.orig_p` to `ocsf.evidence.src_endpoint.port` + enabled: true + sources: + - id.orig_p + sourceType: attribute + target: ocsf.evidence.src_endpoint.port + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.resp_h` to `ocsf.evidence.dst_endpoint.ip` + enabled: true + sources: + - id.resp_h + sourceType: attribute + target: ocsf.evidence.dst_endpoint.ip + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.resp_p` to `ocsf.evidence.dst_endpoint.port` + enabled: true + sources: + - id.resp_p + sourceType: attribute + target: ocsf.evidence.dst_endpoint.port + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Move ocsf.evidence into ocsf.evidences array + enabled: true + operation: + source: ocsf.evidence + target: ocsf.evidences + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 2004 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "*" + name: Create + id: 1 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-remapper + name: Map `ocsf.evidences` to `ocsf.evidences` + sources: + - ocsf.evidences + target: ocsf.evidences + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `note` to `ocsf.finding_info.title` + sources: + - note + target: ocsf.finding_info.title + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.finding_info.uid` + sources: + - uid + target: ocsf.finding_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.is_alert` to `ocsf.is_alert` + sources: + - ocsf.is_alert + target: ocsf.is_alert + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `note` to `ocsf.metadata.event_code` + sources: + - note + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "@severity.id:1" + name: Informational + id: 1 + - filter: + query: "@severity.id:2" + name: Low + id: 2 + - filter: + query: "@severity.id:3" + name: Medium + id: 3 + - filter: + query: "@severity.id:4" + name: High + id: 4 + - filter: + query: "@severity.id:5" + name: Critical + id: 5 + - filter: + query: "*" + name: Other + id: 99 + targets: + name: ocsf.severity + id: ocsf.severity_id + fallback: + values: + ocsf.severity: Other + ocsf.severity_id: "99" + sources: + ocsf.severity: + - severity.name + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "*" + name: New + id: 1 + targets: + name: ocsf.status + id: ocsf.status_id + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: Detection Finding + classUid: 2004 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Detection Finding [2004] - Suricata + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:suricata_corelight" + processors: + - type: string-builder-processor + name: Override product name for Suricata + enabled: true + template: Suricata + target: ocsf.metadata.product.name + replaceMissing: false + - type: string-builder-processor + name: Set log provider + enabled: true + template: Corelight + target: ocsf.metadata.log_provider + replaceMissing: false + - type: string-builder-processor + name: Set is_alert to boolean true + enabled: true + template: "true" + target: ocsf.is_alert + replaceMissing: false + - type: grok-parser + name: Convert ocsf.is_alert string to boolean + enabled: true + source: ocsf.is_alert + samples: + - "true" + grok: + supportRules: "" + matchRules: "to_bool %{boolean(\"true\",\"false\"):ocsf.is_alert}" + - type: attribute-remapper + name: Map `id.orig_h` to `ocsf.evidence.src_endpoint.ip` + enabled: true + sources: + - id.orig_h + sourceType: attribute + target: ocsf.evidence.src_endpoint.ip + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.orig_p` to `ocsf.evidence.src_endpoint.port` + enabled: true + sources: + - id.orig_p + sourceType: attribute + target: ocsf.evidence.src_endpoint.port + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.resp_h` to `ocsf.evidence.dst_endpoint.ip` + enabled: true + sources: + - id.resp_h + sourceType: attribute + target: ocsf.evidence.dst_endpoint.ip + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: Map `id.resp_p` to `ocsf.evidence.dst_endpoint.port` + enabled: true + sources: + - id.resp_p + sourceType: attribute + target: ocsf.evidence.dst_endpoint.port + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Move ocsf.evidence into ocsf.evidences array + enabled: true + operation: + source: ocsf.evidence + target: ocsf.evidences + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 2004 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "*" + name: Create + id: 1 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.confidence_id + categories: + - filter: + query: "@alert.metadata:\"confidence:High\"" + name: High + id: 3 + - filter: + query: "@alert.metadata:\"confidence:Medium\"" + name: Medium + id: 2 + - filter: + query: "@alert.metadata:\"confidence:Low\"" + name: Low + id: 1 + targets: + name: ocsf.confidence + id: ocsf.confidence_id + - type: schema-remapper + name: Map `ocsf.evidences` to `ocsf.evidences` + sources: + - ocsf.evidences + target: ocsf.evidences + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `alert.signature` to `ocsf.finding_info.analytic.name` + sources: + - alert.signature + target: ocsf.finding_info.analytic.name + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.finding_info.analytic.type_id + categories: + - filter: + query: "*" + name: Rule + id: 1 + targets: + name: ocsf.finding_info.analytic.type + id: ocsf.finding_info.analytic.type_id + - type: schema-remapper + name: Map `alert.signature_id` to `ocsf.finding_info.analytic.uid` + sources: + - alert.signature_id + target: ocsf.finding_info.analytic.uid + preserveSource: true + overrideOnConflict: true + targetFormat: string + - type: schema-remapper + name: Map `alert.signature` to `ocsf.finding_info.title` + sources: + - alert.signature + target: ocsf.finding_info.title + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.finding_info.uid` + sources: + - uid + target: ocsf.finding_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `suri_id` to `ocsf.finding_info.uid_alt` + sources: + - suri_id + target: ocsf.finding_info.uid_alt + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.is_alert` to `ocsf.is_alert` + sources: + - ocsf.is_alert + target: ocsf.is_alert + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `alert.signature_id` to `ocsf.metadata.event_code` + sources: + - alert.signature_id + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + targetFormat: string + - type: schema-remapper + name: Map `ocsf.metadata.log_provider` to `ocsf.metadata.log_provider` + sources: + - ocsf.metadata.log_provider + target: ocsf.metadata.log_provider + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "@alert.severity:1" + name: High + id: 4 + - filter: + query: "@alert.severity:2" + name: Medium + id: 3 + - filter: + query: "@alert.severity:3" + name: Low + id: 2 + - filter: + query: "*" + name: Other + id: 99 + targets: + name: ocsf.severity + id: ocsf.severity_id + fallback: + values: + ocsf.severity: Other + ocsf.severity_id: "99" + sources: + ocsf.severity: + - alert_severity + - type: schema-remapper + name: Map `alert.action` to `ocsf.status_detail` + sources: + - alert.action + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: Detection Finding + classUid: 2004 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Network Activity [4001] - conn + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(conn OR conn_long OR conn_red)" + processors: + - type: arithmetic-processor + name: Calculate total bytes + enabled: true + expression: (orig_bytes + resp_bytes) + target: ocsf.traffic.bytes + isReplaceMissing: false + - type: arithmetic-processor + name: Calculate total packets + enabled: true + expression: (orig_pkts + resp_pkts) + target: ocsf.traffic.packets + isReplaceMissing: false + - type: arithmetic-processor + name: Convert duration to milliseconds + enabled: true + expression: duration * 1000 + target: ocsf.duration + isReplaceMissing: false + - type: schema-processor + name: Apply OCSF schema for 4001 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@conn_state:(SF OR RSTO OR RSTR OR RSTRH OR SH OR SHR)" + name: Close + id: 2 + - filter: + query: "@conn_state:(S0 OR RSTOS0)" + name: Fail + id: 4 + - filter: + query: "@conn_state:REJ" + name: Refuse + id: 5 + - filter: + query: "@conn_state:(OTH OR S1 OR S2 OR S3)" + name: Traffic + id: 6 + - filter: + query: "@conn_state:*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - conn_state + - type: schema-category-mapper + name: ocsf.connection_info.boundary_id + categories: + - filter: + query: "@local_orig:true @local_resp:true" + name: Localhost + id: 1 + - filter: + query: "(@local_orig:true @local_resp:false) OR (@local_orig:false @local_resp:true)" + name: External + id: 3 + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.connection_info.boundary + id: ocsf.connection_info.boundary_id + - type: schema-remapper + name: Map `community_id` to `ocsf.connection_info.community_uid` + sources: + - community_id + target: ocsf.connection_info.community_uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.connection_info.direction_id + categories: + - filter: + query: "@local_orig:true @local_resp:false" + name: Outbound + id: 2 + - filter: + query: "@local_orig:false @local_resp:true" + name: Inbound + id: 1 + - filter: + query: "@local_orig:true @local_resp:true" + name: Lateral + id: 3 + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.connection_info.direction + id: ocsf.connection_info.direction_id + - type: schema-remapper + name: Map `history` to `ocsf.connection_info.flag_history` + sources: + - history + target: ocsf.connection_info.flag_history + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `proto` to `ocsf.connection_info.protocol_name` + sources: + - proto + target: ocsf.connection_info.protocol_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.connection_info.uid` + sources: + - uid + target: ocsf.connection_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ocsf.duration` to `ocsf.duration` + sources: + - ocsf.duration + target: ocsf.duration + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `conn_state` to `ocsf.status_detail` + sources: + - conn_state + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@conn_state:(SF OR S1 OR S2 OR S3 OR OTH OR RSTO OR RSTRH OR SH OR SHR)" + name: Success + id: 1 + - filter: + query: "@conn_state:(S0 OR RSTOS0 OR RSTR OR REJ)" + name: Failure + id: 2 + - filter: + query: "*" + name: Other + id: 99 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: "99" + sources: + ocsf.status: + - conn_state + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ocsf.traffic.bytes` to `ocsf.traffic.bytes` + sources: + - ocsf.traffic.bytes + target: ocsf.traffic.bytes + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `resp_bytes` to `ocsf.traffic.bytes_in` + sources: + - resp_bytes + target: ocsf.traffic.bytes_in + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `missed_bytes` to `ocsf.traffic.bytes_missed` + sources: + - missed_bytes + target: ocsf.traffic.bytes_missed + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `orig_bytes` to `ocsf.traffic.bytes_out` + sources: + - orig_bytes + target: ocsf.traffic.bytes_out + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ocsf.traffic.packets` to `ocsf.traffic.packets` + sources: + - ocsf.traffic.packets + target: ocsf.traffic.packets + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `resp_pkts` to `ocsf.traffic.packets_in` + sources: + - resp_pkts + target: ocsf.traffic.packets_in + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `orig_pkts` to `ocsf.traffic.packets_out` + sources: + - orig_pkts + target: ocsf.traffic.packets_out + preserveSource: true + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: Network Activity + classUid: 4001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Network Activity [4001] - ssl + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(ssl OR ssl_red)" + processors: + - type: string-builder-processor + name: Set JA3 hash algorithm name + enabled: true + template: MD5 + target: ocsf.tls.ja3_hash.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set JA3 hash algorithm id + enabled: true + template: "1" + target: ocsf.tls.ja3_hash.algorithm_id + replaceMissing: false + - type: string-builder-processor + name: Set JA3S hash algorithm name + enabled: true + template: MD5 + target: ocsf.tls.ja3s_hash.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set JA3S hash algorithm id + enabled: true + template: "1" + target: ocsf.tls.ja3s_hash.algorithm_id + replaceMissing: false + - type: schema-processor + name: Apply OCSF schema for 4001 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@established:true" + name: Open + id: 1 + - filter: + query: "@established:false" + name: Fail + id: 4 + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-remapper + name: Map `server_name` to `ocsf.dst_endpoint.hostname` + sources: + - server_name + target: ocsf.dst_endpoint.hostname + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ts` to `ocsf.metadata.original_time` + sources: + - ts + target: ocsf.metadata.original_time + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@established:true" + name: Success + id: 1 + - filter: + query: "@established:false" + name: Failure + id: 2 + targets: + name: ocsf.status + id: ocsf.status_id + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `cipher` to `ocsf.tls.cipher` + sources: + - cipher + target: ocsf.tls.cipher + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.tls.ja3_hash.algorithm` to `ocsf.tls.ja3_hash.algorithm` + sources: + - ocsf.tls.ja3_hash.algorithm + target: ocsf.tls.ja3_hash.algorithm + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.tls.ja3_hash.algorithm_id` to `ocsf.tls.ja3_hash.algorithm_id` + sources: + - ocsf.tls.ja3_hash.algorithm_id + target: ocsf.tls.ja3_hash.algorithm_id + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ja3` to `ocsf.tls.ja3_hash.value` + sources: + - ja3 + target: ocsf.tls.ja3_hash.value + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.tls.ja3s_hash.algorithm` to `ocsf.tls.ja3s_hash.algorithm` + sources: + - ocsf.tls.ja3s_hash.algorithm + target: ocsf.tls.ja3s_hash.algorithm + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.tls.ja3s_hash.algorithm_id` to `ocsf.tls.ja3s_hash.algorithm_id` + sources: + - ocsf.tls.ja3s_hash.algorithm_id + target: ocsf.tls.ja3s_hash.algorithm_id + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `ja3s` to `ocsf.tls.ja3s_hash.value` + sources: + - ja3s + target: ocsf.tls.ja3s_hash.value + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `server_name` to `ocsf.tls.sni` + sources: + - server_name + target: ocsf.tls.sni + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `version` to `ocsf.tls.version` + sources: + - version + target: ocsf.tls.version + preserveSource: true + overrideOnConflict: true + schema: + schemaType: ocsf + version: 1.5.0 + className: Network Activity + classUid: 4001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class Network Activity [4001] - weird + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:weird_red" + processors: + - type: schema-processor + name: Apply OCSF schema for 4001 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "*" + name: Traffic + id: 6 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.connection_info.direction_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.connection_info.direction + id: ocsf.connection_info.direction_id + - type: schema-remapper + name: Map `source` to `ocsf.connection_info.protocol_name` + sources: + - source + target: ocsf.connection_info.protocol_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.connection_info.uid` + sources: + - uid + target: ocsf.connection_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `name` to `ocsf.message` + sources: + - name + target: ocsf.message + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `name` to `ocsf.metadata.event_code` + sources: + - name + target: ocsf.metadata.event_code + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `name` to `ocsf.status_detail` + sources: + - name + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: Network Activity + classUid: 4001 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class HTTP Activity [4002] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(http OR http_red)" + processors: + - type: schema-processor + name: Apply OCSF schema for 4002 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@http.method:CONNECT" + name: Connect + id: 1 + - filter: + query: "@http.method:DELETE" + name: Delete + id: 2 + - filter: + query: "@http.method:GET" + name: Get + id: 3 + - filter: + query: "@http.method:HEAD" + name: Head + id: 4 + - filter: + query: "@http.method:OPTIONS" + name: Options + id: 5 + - filter: + query: "@http.method:POST" + name: Post + id: 6 + - filter: + query: "@http.method:PUT" + name: Put + id: 7 + - filter: + query: "@http.method:TRACE" + name: Trace + id: 8 + - filter: + query: "*" + name: Other + id: 99 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + fallback: + values: + ocsf.activity_name: Other + ocsf.activity_id: "99" + sources: + ocsf.activity_name: + - http.method + - type: schema-category-mapper + name: ocsf.connection_info.direction_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.connection_info.direction + id: ocsf.connection_info.direction_id + - type: schema-remapper + name: Map `uid` to `ocsf.connection_info.uid` + sources: + - uid + target: ocsf.connection_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `request_body_len` to `ocsf.http_request.body_length` + sources: + - request_body_len + target: ocsf.http_request.body_length + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `method` to `ocsf.http_request.http_method` + sources: + - method + target: ocsf.http_request.http_method + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.http_request.uid` + sources: + - uid + target: ocsf.http_request.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `host` to `ocsf.http_request.url.hostname` + sources: + - host + target: ocsf.http_request.url.hostname + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uri` to `ocsf.http_request.url.path` + sources: + - uri + target: ocsf.http_request.url.path + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.http_request.url.port` + sources: + - id.resp_p + target: ocsf.http_request.url.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `http_url` to `ocsf.http_request.url.url_string` + sources: + - http_url + target: ocsf.http_request.url.url_string + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `user_agent` to `ocsf.http_request.user_agent` + sources: + - user_agent + target: ocsf.http_request.user_agent + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `version` to `ocsf.http_request.version` + sources: + - version + target: ocsf.http_request.version + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `response_body_len` to `ocsf.http_response.body_length` + sources: + - response_body_len + target: ocsf.http_response.body_length + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `status_code` to `ocsf.http_response.code` + sources: + - status_code + target: ocsf.http_response.code + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `status_msg` to `ocsf.http_response.message` + sources: + - status_msg + target: ocsf.http_response.message + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "@http.status_code:[500 TO 599]" + name: High + id: 4 + - filter: + query: "@http.status_code:[400 TO 499]" + name: Medium + id: 3 + - filter: + query: "@http.status_code:[300 TO 399]" + name: Low + id: 2 + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `status_msg` to `ocsf.status_detail` + sources: + - status_msg + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@http.status_code:[200 TO 299]" + name: Success + id: 1 + - filter: + query: "@http.status_code:[400 TO 599]" + name: Failure + id: 2 + - filter: + query: "*" + name: Other + id: 99 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: "99" + sources: + ocsf.status: + - status_msg + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: HTTP Activity + classUid: 4002 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class DNS Activity [4003] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(dns OR dns_red)" + processors: + - type: string-builder-processor + name: Stringify answers into ocsf.answer + enabled: true + template: "%{answers}" + target: ocsf.answer + replaceMissing: false + - type: grok-parser + name: Extract first answer into ocsf.answer.rdata + enabled: true + source: ocsf.answer + samples: + - "185.64.148.0" + - "185.64.148.0,8.8.8.8" + grok: + supportRules: "" + matchRules: 'a %{data:ocsf.answer.rdata}(,%{data})?' + - type: array-processor + name: Append ocsf.answer into ocsf.answers array + enabled: true + operation: + source: ocsf.answer + target: ocsf.answers + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 4003 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@rcode_name:*" + name: Response + id: 2 + - filter: + query: "-@rcode_name:*" + name: Query + id: 1 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-category-mapper + name: ocsf.connection_info.direction_id + categories: + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.connection_info.direction + id: ocsf.connection_info.direction_id + - type: schema-remapper + name: Map `proto` to `ocsf.connection_info.protocol_name` + sources: + - proto + target: ocsf.connection_info.protocol_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.connection_info.uid` + sources: + - uid + target: ocsf.connection_info.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_h` to `ocsf.dst_endpoint.ip` + sources: + - id.resp_h + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.resp_p` to `ocsf.dst_endpoint.port` + sources: + - id.resp_p + target: ocsf.dst_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `query`, `rcode_name` to `ocsf.message` + sources: + - query + - rcode_name + target: ocsf.message + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `qclass_name` to `ocsf.query.class` + sources: + - qclass_name + target: ocsf.query.class + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `domain`, `query` to `ocsf.query.hostname` + sources: + - domain + - query + target: ocsf.query.hostname + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `qtype_name` to `ocsf.query.type` + sources: + - qtype_name + target: ocsf.query.type + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.answers` to `ocsf.answers` + sources: + - ocsf.answers + target: ocsf.answers + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.rcode_id + categories: + - filter: + query: "@rcode:0" + name: NoError + id: 0 + - filter: + query: "@rcode:1" + name: FormError + id: 1 + - filter: + query: "@rcode:2" + name: ServError + id: 2 + - filter: + query: "@rcode:3" + name: NXDomain + id: 3 + - filter: + query: "@rcode:4" + name: NotImp + id: 4 + - filter: + query: "@rcode:5" + name: Refused + id: 5 + - filter: + query: "@rcode:6" + name: YXDomain + id: 6 + - filter: + query: "@rcode:7" + name: YXRRSet + id: 7 + - filter: + query: "@rcode:8" + name: NXRRSet + id: 8 + - filter: + query: "@rcode:9" + name: NotAuth + id: 9 + - filter: + query: "@rcode:10" + name: NotZone + id: 10 + - filter: + query: "@rcode:11" + name: DSOTYPENI + id: 11 + - filter: + query: "@rcode:16" + name: BADSIG_VERS + id: 16 + - filter: + query: "@rcode:17" + name: BADKEY + id: 17 + - filter: + query: "@rcode:18" + name: BADTIME + id: 18 + - filter: + query: "@rcode:19" + name: BADMODE + id: 19 + - filter: + query: "@rcode:20" + name: BADNAME + id: 20 + - filter: + query: "@rcode:21" + name: BADALG + id: 21 + - filter: + query: "@rcode:22" + name: BADTRUNC + id: 22 + - filter: + query: "@rcode:23" + name: BADCOOKIE + id: 23 + targets: + name: ocsf.rcode + id: ocsf.rcode_id + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `id.orig_h` to `ocsf.src_endpoint.ip` + sources: + - id.orig_h + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `id.orig_p` to `ocsf.src_endpoint.port` + sources: + - id.orig_p + target: ocsf.src_endpoint.port + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-remapper + name: Map `rcode_name` to `ocsf.status_detail` + sources: + - rcode_name + target: ocsf.status_detail + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "(@rcode:0 OR @dns.flags.rcode:NOERROR)" + name: Success + id: 1 + - filter: + query: "(@rcode:3 OR @dns.flags.rcode:NXDOMAIN OR @rejected:true)" + name: Failure + id: 2 + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.status + id: ocsf.status_id + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + targetFormat: integer + schema: + schemaType: ocsf + version: 1.5.0 + className: DNS Activity + classUid: 4003 + extensions: [] + profiles: [] + - type: pipeline + name: OCSF sub pipeline for class File Hosting Activity [6006] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@_path:(files OR files_red)" + processors: + - type: string-builder-processor + name: Stringify tx_hosts into ocsf.src_endpoint + enabled: true + template: "%{tx_hosts}" + target: ocsf.src_endpoint + replaceMissing: false + - type: string-builder-processor + name: Stringify rx_hosts into ocsf.dst_endpoint + enabled: true + template: "%{rx_hosts}" + target: ocsf.dst_endpoint + replaceMissing: false + - type: grok-parser + name: Extract first IP from tx_hosts + enabled: true + source: ocsf.src_endpoint + samples: + - "10.104.10.60" + - "10.104.10.60,10.104.10.61" + grok: + supportRules: "" + matchRules: 'g %{ip:ocsf.src_endpoint.ip}(,%{data})?' + - type: grok-parser + name: Extract first IP from rx_hosts + enabled: true + source: ocsf.dst_endpoint + samples: + - "10.104.10.65" + - "10.104.10.65,10.104.10.66" + grok: + supportRules: "" + matchRules: 'g %{ip:ocsf.dst_endpoint.ip}(,%{data})?' + - type: string-builder-processor + name: Set MD5 algorithm name + enabled: true + template: MD5 + target: tmp_md5.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set MD5 algorithm id + enabled: true + template: "1" + target: tmp_md5.algorithm_id + replaceMissing: false + - type: grok-parser + name: Coerce tmp_md5.algorithm_id to integer + enabled: true + source: tmp_md5.algorithm_id + samples: + - "1" + grok: + supportRules: "" + matchRules: "to_int %{integer:tmp_md5.algorithm_id}" + - type: attribute-remapper + name: Map `md5` to `tmp_md5.value` + enabled: true + sources: + - md5 + sourceType: attribute + target: tmp_md5.value + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Append tmp_md5 to ocsf.file.hashes + enabled: true + operation: + source: tmp_md5 + target: ocsf.file.hashes + preserveSource: false + type: append + - type: string-builder-processor + name: Set SHA1 algorithm name + enabled: true + template: SHA-1 + target: tmp_sha1.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set SHA1 algorithm id + enabled: true + template: "2" + target: tmp_sha1.algorithm_id + replaceMissing: false + - type: grok-parser + name: Coerce tmp_sha1.algorithm_id to integer + enabled: true + source: tmp_sha1.algorithm_id + samples: + - "2" + grok: + supportRules: "" + matchRules: "to_int %{integer:tmp_sha1.algorithm_id}" + - type: attribute-remapper + name: Map `sha1` to `tmp_sha1.value` + enabled: true + sources: + - sha1 + sourceType: attribute + target: tmp_sha1.value + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Append tmp_sha1 to ocsf.file.hashes + enabled: true + operation: + source: tmp_sha1 + target: ocsf.file.hashes + preserveSource: false + type: append + - type: string-builder-processor + name: Set SHA256 algorithm name + enabled: true + template: SHA-256 + target: tmp_sha256.algorithm + replaceMissing: false + - type: string-builder-processor + name: Set SHA256 algorithm id + enabled: true + template: "3" + target: tmp_sha256.algorithm_id + replaceMissing: false + - type: grok-parser + name: Coerce tmp_sha256.algorithm_id to integer + enabled: true + source: tmp_sha256.algorithm_id + samples: + - "3" + grok: + supportRules: "" + matchRules: "to_int %{integer:tmp_sha256.algorithm_id}" + - type: attribute-remapper + name: Map `sha256` to `tmp_sha256.value` + enabled: true + sources: + - sha256 + sourceType: attribute + target: tmp_sha256.value + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: array-processor + name: Append tmp_sha256 to ocsf.file.hashes + enabled: true + operation: + source: tmp_sha256 + target: ocsf.file.hashes + preserveSource: false + type: append + - type: schema-processor + name: Apply OCSF schema for 6006 + enabled: true + mappers: + - type: schema-category-mapper + name: ocsf.activity_id + categories: + - filter: + query: "@is_orig:true" + name: Upload + id: 1 + - filter: + query: "@is_orig:false" + name: Download + id: 2 + - filter: + query: "*" + name: Unknown + id: 0 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + - type: schema-remapper + name: Map `uid` to `ocsf.actor.session.uid` + sources: + - uid + target: ocsf.actor.session.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.dst_endpoint.ip` to `ocsf.dst_endpoint.ip` + sources: + - ocsf.dst_endpoint.ip + target: ocsf.dst_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.file.hashes` to `ocsf.file.hashes` + sources: + - ocsf.file.hashes + target: ocsf.file.hashes + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `mime_type` to `ocsf.file.mime_type` + sources: + - mime_type + target: ocsf.file.mime_type + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `filename`, `fuid` to `ocsf.file.name` + sources: + - filename + - fuid + target: ocsf.file.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `seen_bytes`, `total_bytes` to `ocsf.file.size` + sources: + - seen_bytes + - total_bytes + target: ocsf.file.size + preserveSource: true + overrideOnConflict: true + targetFormat: integer + - type: schema-category-mapper + name: ocsf.file.type_id + categories: + - filter: + query: "*" + name: Regular File + id: 1 + targets: + name: ocsf.file.type_id + id: ocsf.file.type_id + - type: schema-remapper + name: Map `fuid` to `ocsf.file.uid` + sources: + - fuid + target: ocsf.file.uid + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + target: ocsf.metadata.product.name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + target: ocsf.metadata.product.vendor_name + preserveSource: true + overrideOnConflict: true + - type: schema-remapper + name: Map `uid` to `ocsf.metadata.uid` + sources: + - uid + target: ocsf.metadata.uid + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.severity_id + categories: + - filter: + query: "*" + name: Informational + id: 1 + targets: + name: ocsf.severity + id: ocsf.severity_id + - type: schema-remapper + name: Map `ocsf.src_endpoint.ip` to `ocsf.src_endpoint.ip` + sources: + - ocsf.src_endpoint.ip + target: ocsf.src_endpoint.ip + preserveSource: true + overrideOnConflict: true + - type: schema-category-mapper + name: ocsf.status_id + categories: + - filter: + query: "@timedout:false" + name: Success + id: 1 + - filter: + query: "@timedout:true" + name: Failure + id: 2 + - filter: + query: "@timedout:*" + name: Other + id: 99 + targets: + name: ocsf.status + id: ocsf.status_id + fallback: + values: + ocsf.status: Other + ocsf.status_id: 99 + sources: + ocsf.status: + - timedout + - type: schema-remapper + name: Map `ocsf.time` to `ocsf.time` + sources: + - ocsf.time + target: ocsf.time + preserveSource: false + overrideOnConflict: true + schema: + schemaType: ocsf + version: 1.5.0 + className: File Hosting Activity + classUid: 6006 + extensions: [] + profiles: [] diff --git a/zeek/assets/logs/zeek_tests.yaml b/zeek/assets/logs/zeek_tests.yaml index dc27f90be8adf..55e841d90bd83 100644 --- a/zeek/assets/logs/zeek_tests.yaml +++ b/zeek/assets/logs/zeek_tests.yaml @@ -18,7 +18,7 @@ tests: message: "<134>Dec 26 01:35:11 machine-name {\"_path\":\"capture_loss\",\"_write_ts\":\"2023-12-12T05:52:50.756358Z\",\"ts\":\"2023-12-12T05:52:32.763303Z\",\"ts_delta\":15.235642194747925,\"peer\":\"zeek\",\"gaps\":3,\"acks\":316,\"percent_lost\":0.9493670886075949}" service: "capture_loss" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: |- @@ -73,6 +73,57 @@ tests: duration_sec: 3.0111899375915527 missed_bytes: 0 proto: "icmp" + ocsf: + severity: Informational + activity_name: Traffic + metadata: + uid: CcdWSj20NUmxZowq93 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 3 + ip: 185.64.148.0 + status_detail: OTH + duration: 3011 + status_id: 1 + connection_info: + boundary: Localhost + boundary_id: 1 + uid: CcdWSj20NUmxZowq93 + direction_id: 3 + protocol_name: icmp + direction: Lateral + class_uid: 4001 + activity_id: 6 + time: 1702360352763 + dst_endpoint: + port: 10 + ip: 185.64.148.0 + severity_id: 1 + class_name: Network Activity + traffic: + bytes_out: 234 + bytes_in: 0 + packets_out: 3 + bytes: 234 + packets_in: 0 + packets: 3 + bytes_missed: 0 + status: Success + orig_bytes: 234 + missed_bytes: 0 + duration: 3.0111899375915527 + resp_bytes: 0 + proto: icmp + id: + orig_p: 3 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 10 message: |- { "resp_pkts" : 0, @@ -100,7 +151,7 @@ tests: } service: "conn" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: |- @@ -153,17 +204,74 @@ tests: ip: "185.64.148.0" port: 80 orig_fuids: - - "FKhxp22QnZua8NWY2" + - "FKhxp22QnZua8NWY2" orig_mime_types: - - "application/ocsp-request" + - "application/ocsp-request" resp_fuids: - - "F9Zqr71YzMZPVZ4dbe" + - "F9Zqr71YzMZPVZ4dbe" resp_mime_types: - - "application/ocsp-response" + - "application/ocsp-response" status_msg: "OK" trans_depth: 1 ts: "2023-12-12T05:52:32.763303Z" uid: "CBQBvs1hob384mp2lb" + ocsf: + http_response: + code: 200 + message: OK + body_length: 1434 + severity: Informational + activity_name: Post + metadata: + uid: CBQBvs1hob384mp2lb + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 47210 + ip: 185.64.148.0 + status_detail: OK + status_id: 1 + connection_info: + uid: CBQBvs1hob384mp2lb + direction_id: 0 + direction: Unknown + class_uid: 4002 + activity_id: 6 + http_request: + uid: CBQBvs1hob384mp2lb + http_method: POST + version: '1.1' + url: + path: /rootr3 + hostname: host.com + port: 80 + url_string: http://host.com/rootr3 + user_agent: LIBCURL + body_length: 83 + time: 1702360352763 + dst_endpoint: + port: 80 + ip: 185.64.148.0 + severity_id: 1 + class_name: HTTP Activity + status: Success + status_code: 200 + method: POST + request_body_len: 83 + uri: /rootr3 + version: '1.1' + host: host.com + id: + orig_p: 47210 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 80 + response_body_len: 1434 + user_agent: LIBCURL message: |- { "status_code" : 200, @@ -193,7 +301,7 @@ tests: } service: "http" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: |- @@ -203,7 +311,7 @@ tests: "rejected" : false, "query" : "win2k16-1-159", "_write_ts" : "2023-12-12T05:52:50.756358Z", - "answers" : [ "185.64.148.0" ], + "answers" : [ "185.64.148.0", "8.8.8.8" ], "trans_id" : 38706, "rcode" : 0, "_path" : "dns", @@ -229,14 +337,15 @@ tests: RD: false TC: false TTLs: - - 30.0 + - 30 Z: 0 _path: "dns" _write_ts: "2023-12-12T05:52:50.756358Z" dns: answer: name: - - "185.64.148.0" + - 185.64.148.0 + - 8.8.8.8 flags: rcode: "NOERROR" id: 38706 @@ -272,6 +381,55 @@ tests: uid: "CsOSdHqRMu62rNs31" zeek: proto: "udp" + ocsf: + severity: Informational + activity_name: Response + metadata: + uid: CsOSdHqRMu62rNs31 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 58013 + ip: 185.64.148.0 + query: + hostname: win2k16-1-159 + rcode: NoError + message: win2k16-1-159 + status_detail: NOERROR + rcode_id: 0 + status_id: 1 + connection_info: + uid: CsOSdHqRMu62rNs31 + protocol_name: udp + direction_id: 0 + direction: Unknown + class_uid: 4003 + activity_id: 2 + time: 1702360352763 + dst_endpoint: + port: 5355 + ip: 185.64.148.0 + severity_id: 1 + class_name: DNS Activity + status: Success + answers: + - rdata: 185.64.148.0 + query: win2k16-1-159 + answers: + - 185.64.148.0 + - 8.8.8.8 + trans_id: 38706 + rcode_name: NOERROR + proto: udp + id: + orig_p: 58013 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 5355 message: |- { "AA" : false, @@ -279,7 +437,7 @@ tests: "rejected" : false, "query" : "win2k16-1-159", "_write_ts" : "2023-12-12T05:52:50.756358Z", - "answers" : [ "185.64.148.0" ], + "answers" : [ "185.64.148.0", "8.8.8.8" ], "trans_id" : 38706, "rcode" : 0, "_path" : "dns", @@ -300,7 +458,7 @@ tests: } service: "dns" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: |- @@ -324,7 +482,7 @@ tests: service: "reporter" status: "info" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1702360352763 - sample: "<134>Jan 12 18:13:46 machine-name {\"_path\":\"datared\",\"_write_ts\":\"2024-01-13T00:13:42.817478Z\",\"ts\":\"2024-01-13T00:13:42.817478Z\",\"conn_red\":1310,\"conn_total\":18460,\"dns_red\":1141,\"dns_total\":1604,\"files_red\":313,\"files_total\":796,\"http_red\":140,\"http_total\":140,\"ssl_red\":215,\"ssl_total\":227,\"weird_red\":20,\"weird_total\":20}" @@ -350,52 +508,8 @@ tests: message: "<134>Jan 12 18:13:46 machine-name {\"_path\":\"datared\",\"_write_ts\":\"2024-01-13T00:13:42.817478Z\",\"ts\":\"2024-01-13T00:13:42.817478Z\",\"conn_red\":1310,\"conn_total\":18460,\"dns_red\":1141,\"dns_total\":1604,\"files_red\":313,\"files_total\":796,\"http_red\":140,\"http_total\":140,\"ssl_red\":215,\"ssl_total\":227,\"weird_red\":20,\"weird_total\":20}" service: "datared" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1705104822817 - - - sample: "<134>Mar 6 20:22:19 machine-name {\"_path\":\"files_red\",\"_write_ts\":\"2024-03-07T02:22:18.145479Z\",\"ts\":[\"2024-03-07T02:12:12.099810Z\"],\"fuid\":\"FJcjJV2mrRbxT0TWi\",\"tx_hosts\":[\"10.10.10.10\"],\"rx_hosts\":[\"20.20.20.20\"],\"conn_uids\":[\"CALvyD1AyopngTm5Xh\"],\"source\":\"HTTP\",\"depth\":0,\"analyzers\":[\"SHA1\",\"MD5\",\"DATA_EVENT\",\"SHA256\"],\"local_orig\":true,\"is_orig\":false,\"seen_bytes\":163,\"total_bytes\":163,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"extracted\":[],\"md5\":\"e335c05220a3858e858d2026071e3se2\",\"sha1\":\"8514de756fa993adb449c282affe84c752bab495\",\"sha256\":\"2733c2e83762f8a9542ebacf40642505fee0247295161ced264b44d2ad6f9456\",\"num\":1}" - service: "corelight" - result: - custom: - _path: "files_red" - _write_ts: "2024-03-07T02:22:18.145479Z" - analyzers: - - "SHA1" - - "MD5" - - "DATA_EVENT" - - "SHA256" - conn_uids: - - "CALvyD1AyopngTm5Xh" - depth: 0 - dest_host: - - "10.10.10.10" - fuid: "FJcjJV2mrRbxT0TWi" - is_orig: false - local_orig: true - md5: "e335c05220a3858e858d2026071e3se2" - num: 1 - orig_host: - - "20.20.20.20" - rx_hosts: - - "20.20.20.20" - sha1: "8514de756fa993adb449c282affe84c752bab495" - sha256: "2733c2e83762f8a9542ebacf40642505fee0247295161ced264b44d2ad6f9456" - source: "HTTP" - timedout: false - ts: - - "2024-03-07T02:12:12.099810Z" - tx_hosts: - - "10.10.10.10" - zeek: - missing_bytes: 0 - overflow_bytes: 0 - seen_bytes: 163 - total_bytes: 163 - message: "<134>Mar 6 20:22:19 machine-name {\"_path\":\"files_red\",\"_write_ts\":\"2024-03-07T02:22:18.145479Z\",\"ts\":[\"2024-03-07T02:12:12.099810Z\"],\"fuid\":\"FJcjJV2mrRbxT0TWi\",\"tx_hosts\":[\"10.10.10.10\"],\"rx_hosts\":[\"20.20.20.20\"],\"conn_uids\":[\"CALvyD1AyopngTm5Xh\"],\"source\":\"HTTP\",\"depth\":0,\"analyzers\":[\"SHA1\",\"MD5\",\"DATA_EVENT\",\"SHA256\"],\"local_orig\":true,\"is_orig\":false,\"seen_bytes\":163,\"total_bytes\":163,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"extracted\":[],\"md5\":\"e335c05220a3858e858d2026071e3se2\",\"sha1\":\"8514de756fa993adb449c282affe84c752bab495\",\"sha256\":\"2733c2e83762f8a9542ebacf40642505fee0247295161ced264b44d2ad6f9456\",\"num\":1}" - service: "files_red" - tags: - - "source:LOGS_SOURCE" - timestamp: 1709778138145 - sample: "<134>Mar 6 22:39:30 machine-name {\"_path\":\"dns_red\",\"_write_ts\":\"2024-03-07T04:39:28.580374Z\",\"ts\":\"2024-03-07T04:38:40.085451Z\",\"uid\":\"CyArTY2KEgcygwMLi1\",\"id.orig_h\":\"10.10.10.10\",\"id.orig_p\":123,\"id.resp_h\":\"20.20.20.20\",\"id.resp_p\":321,\"query\":\"time.missouri.edu\",\"qtype_name\":\"A\",\"num\":4}" service: "corelight" @@ -418,10 +532,49 @@ tests: num: 4 ts: "2024-03-07T04:38:40.085451Z" uid: "CyArTY2KEgcygwMLi1" + ocsf: + severity: Informational + activity_name: Query + metadata: + uid: CyArTY2KEgcygwMLi1 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 123 + ip: 10.10.10.10 + query: + hostname: time.missouri.edu + type: A + message: time.missouri.edu + connection_info: + uid: CyArTY2KEgcygwMLi1 + direction_id: 0 + direction: Unknown + class_uid: 4003 + activity_id: 1 + time: 1709786320085 + dst_endpoint: + port: 321 + ip: 20.20.20.20 + severity_id: 1 + class_name: DNS Activity + status_id: 0 + status: Unknown + qtype_name: A + query: time.missouri.edu + id: + orig_p: 123 + resp_h: 20.20.20.20 + orig_h: 10.10.10.10 + resp_p: 321 message: "<134>Mar 6 22:39:30 machine-name {\"_path\":\"dns_red\",\"_write_ts\":\"2024-03-07T04:39:28.580374Z\",\"ts\":\"2024-03-07T04:38:40.085451Z\",\"uid\":\"CyArTY2KEgcygwMLi1\",\"id.orig_h\":\"10.10.10.10\",\"id.orig_p\":123,\"id.resp_h\":\"20.20.20.20\",\"id.resp_p\":321,\"query\":\"time.missouri.edu\",\"qtype_name\":\"A\",\"num\":4}" service: "dns_red" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1709786320085 - sample: "<134>Nov 19 22:36:37 machine-name {\"_path\":\"etc_viz\",\"_system_name\":\"machine-name\",\"_write_ts\":\"2024-03-28T00:01:04.742415Z\",\"c2s_viz.clr_frac\":0.0,\"c2s_viz.enc_dev\":0.30151134457776363,\"c2s_viz.enc_frac\":1.0,\"c2s_viz.pdu1_enc\":true,\"c2s_viz.size\":198,\"err\":null,\"s2c_viz.clr_frac\":0.0,\"s2c_viz.enc_dev\":0.6957566520492713,\"s2c_viz.enc_frac\":1.0,\"s2c_viz.pdu1_enc\":true,\"s2c_viz.size\":818,\"server_a\":\"10.10.10.10\",\"server_p\":443,\"service\":[],\"ts\":\"2024-03-28T00:00:52.656388Z\",\"uid\":\"CV79JD7onkfdf7WDa\",\"viz_stat\":\"Ee!!\"}" @@ -432,9 +585,9 @@ tests: _system_name: "machine-name" _write_ts: "2024-03-28T00:01:04.742415Z" c2s_viz: - clr_frac: 0.0 + clr_frac: 0 enc_dev: 0.30151134457776363 - enc_frac: 1.0 + enc_frac: 1 pdu1_enc: true size: 198 network: @@ -442,16 +595,874 @@ tests: ip: "10.10.10.10" port: 443 s2c_viz: - clr_frac: 0.0 + clr_frac: 0 enc_dev: 0.6957566520492713 - enc_frac: 1.0 + enc_frac: 1 pdu1_enc: true size: 818 ts: "2024-03-28T00:00:52.656388Z" uid: "CV79JD7onkfdf7WDa" viz_stat: "Ee!!" + server_p: 443 message: "<134>Nov 19 22:36:37 machine-name {\"_path\":\"etc_viz\",\"_system_name\":\"machine-name\",\"_write_ts\":\"2024-03-28T00:01:04.742415Z\",\"c2s_viz.clr_frac\":0.0,\"c2s_viz.enc_dev\":0.30151134457776363,\"c2s_viz.enc_frac\":1.0,\"c2s_viz.pdu1_enc\":true,\"c2s_viz.size\":198,\"err\":null,\"s2c_viz.clr_frac\":0.0,\"s2c_viz.enc_dev\":0.6957566520492713,\"s2c_viz.enc_frac\":1.0,\"s2c_viz.pdu1_enc\":true,\"s2c_viz.size\":818,\"server_a\":\"10.10.10.10\",\"server_p\":443,\"service\":[],\"ts\":\"2024-03-28T00:00:52.656388Z\",\"uid\":\"CV79JD7onkfdf7WDa\",\"viz_stat\":\"Ee!!\"}" service: "etc_viz" tags: - - "source:LOGS_SOURCE" + - "source:LOGS_SOURCE" timestamp: 1711584052656 + - + sample: "<134>May 11 18:47:07 test-system {\"_path\":\"conn\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T18:47:07.850764Z\",\"ts\":\"2026-05-11T18:47:02.848960Z\",\"uid\":\"12345678901234567890\",\"id.orig_h\":\"10.10.50.1\",\"id.orig_p\":60704,\"id.resp_h\":\"10.10.60.6\",\"id.resp_p\":88,\"proto\":\"tcp\",\"service\":\"krb_tcp\",\"orig_bytes\":227,\"resp_bytes\":195,\"conn_state\":\"RSTR\",\"local_orig\":true,\"local_resp\":true,\"missed_bytes\":0,\"history\":\"ShADTdFar\",\"orig_pkts\":5,\"resp_pkts\":7,\"orig_ip_bytes\":666,\"resp_ip_bytes\":499,\"community_id\":\"1:abc123\",\"tunnel_parents\":[\"test\",\"test2\"]}" + service: "corelight" + result: + custom: + resp_pkts: 7 + resp_ip_bytes: 499 + local_orig: true + _write_ts: '2026-05-11T18:47:07.850764Z' + orig_ip_bytes: 666 + zeek: + proto: tcp + missed_bytes: 0 + orig_pkts: 5 + _path: conn + history: ShADTdFar + tunnel_parents: + - test + - test2 + connection_state: Responder sent a RST + network: + bytes_written: 195 + destination: + port: 88 + ip: 10.10.60.6 + client: + port: 60704 + ip: 10.10.50.1 + bytes_read: 227 + local_resp: true + uid: '12345678901234567890' + community_id: 1:abc123 + ocsf: + severity: Informational + activity_name: Close + metadata: + uid: '12345678901234567890' + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 60704 + ip: 10.10.50.1 + status_detail: RSTR + status_id: 2 + connection_info: + boundary: Localhost + boundary_id: 1 + uid: '12345678901234567890' + direction_id: 3 + community_uid: 1:abc123 + flag_history: ShADTdFar + protocol_name: tcp + direction: Lateral + class_uid: 4001 + activity_id: 2 + time: 1778525222848 + dst_endpoint: + port: 88 + ip: 10.10.60.6 + severity_id: 1 + class_name: Network Activity + traffic: + bytes_out: 227 + bytes_in: 195 + packets_out: 5 + bytes: 422 + packets_in: 7 + packets: 12 + bytes_missed: 0 + status: Failure + _system_name: test-system + service: krb_tcp + conn_state: RSTR + ts: '2026-05-11T18:47:02.848960Z' + id: + orig_p: 60704 + resp_h: 10.10.60.6 + orig_h: 10.10.50.1 + resp_p: 88 + orig_bytes: 227 + missed_bytes: 0 + resp_bytes: 195 + proto: tcp + service: conn + message: <134>May 11 18:47:07 test-system {"_path":"conn","_system_name":"test-system","_write_ts":"2026-05-11T18:47:07.850764Z","ts":"2026-05-11T18:47:02.848960Z","uid":"12345678901234567890","id.orig_h":"10.10.50.1","id.orig_p":60704,"id.resp_h":"10.10.60.6","id.resp_p":88,"proto":"tcp","service":"krb_tcp","orig_bytes":227,"resp_bytes":195,"conn_state":"RSTR","local_orig":true,"local_resp":true,"missed_bytes":0,"history":"ShADTdFar","orig_pkts":5,"resp_pkts":7,"orig_ip_bytes":666,"resp_ip_bytes":499,"community_id":"1:abc123","tunnel_parents":["test","test2"]} + tags: + - source:LOGS_SOURCE + timestamp: 1778525222848 + - + sample: "<134>May 11 18:46:03 ndr-pvg3-1 {\"_path\":\"conn_long\",\"_system_name\":\"ndr-pvg3-1\",\"_write_ts\":\"2026-05-11T18:46:03.887017Z\",\"ts\":\"2026-05-11T18:36:03.886935Z\",\"uid\":\"12345678901234568\",\"id.orig_h\":\"10.250.5.75\",\"id.orig_p\":63231,\"id.resp_h\":\"172.64.5.1\",\"id.resp_p\":443,\"id.vlan\":1000,\"proto\":\"tcp\",\"service\":\"ssl\",\"duration\":600.0000820159912,\"orig_bytes\":7071,\"resp_bytes\":18980,\"conn_state\":\"S1\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"ShADadtT\",\"orig_pkts\":32,\"resp_pkts\":37,\"orig_ip_bytes\":8380,\"resp_ip_bytes\":20532,\"community_id\":\"1:def456\",\"corelight_shunted\":false}" + service: "corelight" + result: + custom: + resp_pkts: 37 + corelight_shunted: false + connection_state: Connection established - not terminated + network: + bytes_written: 18980 + destination: + port: 443 + ip: 172.64.5.1 + client: + port: 63231 + ip: 10.250.5.75 + bytes_read: 7071 + local_resp: false + uid: '12345678901234568' + community_id: 1:def456 + ocsf: + severity: Informational + activity_name: Traffic + metadata: + uid: '12345678901234568' + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 63231 + ip: 10.250.5.75 + status_detail: S1 + duration: 600000 + status_id: 1 + connection_info: + boundary: External + boundary_id: 3 + uid: '12345678901234568' + direction_id: 2 + community_uid: 1:def456 + flag_history: ShADadtT + protocol_name: tcp + direction: Outbound + class_uid: 4001 + activity_id: 6 + time: 1778524563886 + dst_endpoint: + port: 443 + ip: 172.64.5.1 + severity_id: 1 + class_name: Network Activity + traffic: + bytes_out: 7071 + bytes_in: 18980 + packets_out: 32 + bytes: 26051 + packets_in: 37 + packets: 69 + bytes_missed: 0 + status: Success + conn_state: S1 + id: + vlan: 1000 + orig_p: 63231 + resp_h: 172.64.5.1 + orig_h: 10.250.5.75 + resp_p: 443 + resp_ip_bytes: 20532 + local_orig: true + _write_ts: '2026-05-11T18:46:03.887017Z' + orig_ip_bytes: 8380 + zeek: + proto: tcp + missed_bytes: 0 + duration_sec: 600.0000820159912 + orig_pkts: 32 + _path: conn_long + history: ShADadtT + _system_name: ndr-pvg3-1 + service: ssl + ts: '2026-05-11T18:36:03.886935Z' + duration: 600.0000820159912 + orig_bytes: 7071 + missed_bytes: 0 + resp_bytes: 18980 + proto: tcp + service: conn_long + message: <134>May 11 18:46:03 ndr-pvg3-1 {"_path":"conn_long","_system_name":"ndr-pvg3-1","_write_ts":"2026-05-11T18:46:03.887017Z","ts":"2026-05-11T18:36:03.886935Z","uid":"12345678901234568","id.orig_h":"10.250.5.75","id.orig_p":63231,"id.resp_h":"172.64.5.1","id.resp_p":443,"id.vlan":1000,"proto":"tcp","service":"ssl","duration":600.0000820159912,"orig_bytes":7071,"resp_bytes":18980,"conn_state":"S1","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"ShADadtT","orig_pkts":32,"resp_pkts":37,"orig_ip_bytes":8380,"resp_ip_bytes":20532,"community_id":"1:def456","corelight_shunted":false} + tags: + - source:LOGS_SOURCE + timestamp: 1778524563886 + - + sample: "<134>Dec 12 05:52:50 machine-name {\"_path\":\"http\",\"_write_ts\":\"2023-12-12T05:52:50.756358Z\",\"ts\":\"2023-12-12T05:52:32.763303Z\",\"uid\":\"CBQBvs1hob384mp2lb\",\"id.orig_h\":\"185.64.148.0\",\"id.orig_p\":47210,\"id.resp_h\":\"185.64.148.0\",\"id.resp_p\":80,\"method\":\"POST\",\"host\":\"host.com\",\"uri\":\"/rootr3\",\"version\":\"1.1\",\"user_agent\":\"LIBCURL\",\"request_body_len\":83,\"response_body_len\":1434,\"status_code\":200,\"status_msg\":\"OK\",\"trans_depth\":1,\"orig_mime_types\":[\"application/ocsp-request\"],\"resp_mime_types\":[\"application/ocsp-response\"],\"orig_fuids\":[\"FKhxp22QnZua8NWY2\"],\"resp_fuids\":[\"F9Zqr71YzMZPVZ4dbe\"]}" + service: "corelight" + result: + custom: + _write_ts: '2023-12-12T05:52:50.756358Z' + _path: http + orig_mime_types: + - application/ocsp-request + http_url: http://host.com/rootr3 + network: + bytes_written: 1434 + destination: + port: 80 + ip: 185.64.148.0 + client: + port: 47210 + ip: 185.64.148.0 + bytes_read: 83 + uid: CBQBvs1hob384mp2lb + resp_mime_types: + - application/ocsp-response + trans_depth: 1 + orig_fuids: + - FKhxp22QnZua8NWY2 + ocsf: + http_response: + code: 200 + message: OK + body_length: 1434 + severity: Informational + activity_name: Post + metadata: + uid: CBQBvs1hob384mp2lb + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 47210 + ip: 185.64.148.0 + status_detail: OK + status_id: 1 + connection_info: + uid: CBQBvs1hob384mp2lb + direction_id: 0 + direction: Unknown + class_uid: 4002 + activity_id: 6 + http_request: + uid: CBQBvs1hob384mp2lb + http_method: POST + version: '1.1' + url: + path: /rootr3 + hostname: host.com + port: 80 + url_string: http://host.com/rootr3 + user_agent: LIBCURL + body_length: 83 + time: 1702360352763 + dst_endpoint: + port: 80 + ip: 185.64.148.0 + severity_id: 1 + class_name: HTTP Activity + status: Success + status_msg: OK + http: + url_details: + path: /rootr3 + host: host.com + status_code: 200 + method: POST + useragent: LIBCURL + version: '1.1' + ts: '2023-12-12T05:52:32.763303Z' + resp_fuids: + - F9Zqr71YzMZPVZ4dbe + status_code: 200 + method: POST + request_body_len: 83 + uri: /rootr3 + version: '1.1' + host: host.com + id: + orig_p: 47210 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 80 + response_body_len: 1434 + user_agent: LIBCURL + service: http + message: <134>Dec 12 05:52:50 machine-name {"_path":"http","_write_ts":"2023-12-12T05:52:50.756358Z","ts":"2023-12-12T05:52:32.763303Z","uid":"CBQBvs1hob384mp2lb","id.orig_h":"185.64.148.0","id.orig_p":47210,"id.resp_h":"185.64.148.0","id.resp_p":80,"method":"POST","host":"host.com","uri":"/rootr3","version":"1.1","user_agent":"LIBCURL","request_body_len":83,"response_body_len":1434,"status_code":200,"status_msg":"OK","trans_depth":1,"orig_mime_types":["application/ocsp-request"],"resp_mime_types":["application/ocsp-response"],"orig_fuids":["FKhxp22QnZua8NWY2"],"resp_fuids":["F9Zqr71YzMZPVZ4dbe"]} + tags: + - source:LOGS_SOURCE + timestamp: 1702360352763 + - + sample: "<134>Dec 12 05:52:50 machine-name {\"_path\":\"dns\",\"_write_ts\":\"2023-12-12T05:52:50.756358Z\",\"ts\":\"2023-12-12T05:52:32.763303Z\",\"uid\":\"CsOSdHqRMu62rNs31\",\"id.orig_h\":\"185.64.148.0\",\"id.orig_p\":58013,\"id.resp_h\":\"185.64.148.0\",\"id.resp_p\":5355,\"proto\":\"udp\",\"trans_id\":38706,\"rcode\":0,\"rcode_name\":\"NOERROR\",\"query\":\"win2k16-1-159\",\"answers\":[\"185.64.148.0\"],\"TTLs\":[30.0],\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"rejected\":false}" + service: "corelight" + result: + custom: + TTLs: + - 30 + AA: false + rejected: false + _write_ts: '2023-12-12T05:52:50.756358Z' + dns: + question: + name: win2k16-1-159 + answer: + name: + - 185.64.148.0 + flags: + rcode: NOERROR + id: 38706 + zeek: + proto: udp + rcode: 0 + _path: dns + TC: false + RA: false + network: + destination: + port: 5355 + ip: 185.64.148.0 + client: + geoip: + continent: + code: EU + name: Europe + country: + name: France + iso_code: FR + subdivision: + name: Île-de-France + iso_code: FR-IDF + city: + name: Paris + timezone: Europe/Paris + ipAddress: 185.64.148.0 + location: + latitude: 48.90654 + longitude: 2.33339 + port: 58013 + ip: 185.64.148.0 + uid: CsOSdHqRMu62rNs31 + RD: false + ocsf: + severity: Informational + activity_name: Response + metadata: + uid: CsOSdHqRMu62rNs31 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 58013 + ip: 185.64.148.0 + query: + hostname: win2k16-1-159 + rcode: NoError + message: win2k16-1-159 + status_detail: NOERROR + rcode_id: 0 + status_id: 1 + connection_info: + uid: CsOSdHqRMu62rNs31 + protocol_name: udp + direction_id: 0 + direction: Unknown + class_uid: 4003 + activity_id: 2 + time: 1702360352763 + dst_endpoint: + port: 5355 + ip: 185.64.148.0 + severity_id: 1 + class_name: DNS Activity + status: Success + answers: + - rdata: 185.64.148.0 + Z: 0 + ts: '2023-12-12T05:52:32.763303Z' + query: win2k16-1-159 + answers: + - 185.64.148.0 + trans_id: 38706 + rcode_name: NOERROR + proto: udp + id: + orig_p: 58013 + resp_h: 185.64.148.0 + orig_h: 185.64.148.0 + resp_p: 5355 + service: dns + message: <134>Dec 12 05:52:50 machine-name {"_path":"dns","_write_ts":"2023-12-12T05:52:50.756358Z","ts":"2023-12-12T05:52:32.763303Z","uid":"CsOSdHqRMu62rNs31","id.orig_h":"185.64.148.0","id.orig_p":58013,"id.resp_h":"185.64.148.0","id.resp_p":5355,"proto":"udp","trans_id":38706,"rcode":0,"rcode_name":"NOERROR","query":"win2k16-1-159","answers":["185.64.148.0"],"TTLs":[30.0],"AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} + tags: + - source:LOGS_SOURCE + timestamp: 1702360352763 + - + sample: "<134>May 11 17:22:02 test-system {\"_path\":\"ssl\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T17:22:02.759122Z\",\"ts\":\"2026-05-11T17:22:02.754532Z\",\"uid\":\"Cxu3mb4NZI2tAscrhb\",\"id.orig_h\":\"10.205.140.1\",\"id.orig_p\":43108,\"id.resp_h\":\"10.205.180.10\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\",\"curve\":\"secp256r1\",\"server_name\":null,\"resumed\":false,\"established\":true,\"ssl_history\":\"CsxknGIti\",\"ja3\":\"c34a54599a1fbaf1786bb6d633545a60\",\"ja3s\":\"03788d8896c247631984a250db971b74\",\"subject\":\"CN=*.org.test.io,O=Test Org,L=Los Angeles,ST=California,C=US\",\"issuer\":\"CN=Test Issuer,OU=Test OU,O=Test Org,C=US\",\"validation_status\":\"ok\",\"cert_chain_fps\":[\"3f5332c4631559ebf68bacbdf27d408g5d904bf5fa8e4f455c5274aae330ca96\",\"8bb2f6883fed289a521ba27c478482950874e167caccec6fc025990c0c46813f\"],\"port\":23367}" + service: "corelight" + result: + custom: + curve: secp256r1 + subject: CN=*.org.test.io,O=Test Org,L=Los Angeles,ST=California,C=US + ssl_history: CsxknGIti + issuer: CN=Test Issuer,OU=Test OU,O=Test Org,C=US + network: + destination: + port: 443 + ip: 10.205.180.10 + client: + port: 43108 + ip: 10.205.140.1 + uid: Cxu3mb4NZI2tAscrhb + ocsf: + severity: Informational + activity_name: Open + metadata: + uid: Cxu3mb4NZI2tAscrhb + product: + name: Zeek + vendor_name: Corelight + original_time: '2026-05-11T17:22:02.754532Z' + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 43108 + ip: 10.205.140.1 + status_id: 1 + class_uid: 4001 + activity_id: 1 + tls: + cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + ja3s_hash: + value: 03788d8896c247631984a250db971b74 + algorithm_id: 1 + algorithm: MD5 + ja3_hash: + value: c34a54599a1fbaf1786bb6d633545a60 + algorithm_id: 1 + algorithm: MD5 + version: TLSv12 + time: 1778520122754 + dst_endpoint: + port: 443 + ip: 10.205.180.10 + severity_id: 1 + class_name: Network Activity + status: Success + cert_chain_fps: + - 3f5332c4631559ebf68bacbdf27d408g5d904bf5fa8e4f455c5274aae330ca96 + - 8bb2f6883fed289a521ba27c478482950874e167caccec6fc025990c0c46813f + ja3: c34a54599a1fbaf1786bb6d633545a60 + cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + established: true + ja3s: 03788d8896c247631984a250db971b74 + _write_ts: '2026-05-11T17:22:02.759122Z' + _path: ssl + version: TLSv12 + _system_name: test-system + port: 23367 + validation_status: ok + resumed: false + ts: '2026-05-11T17:22:02.754532Z' + id: + orig_p: 43108 + resp_h: 10.205.180.10 + orig_h: 10.205.140.1 + resp_p: 443 + service: ssl + message: <134>May 11 17:22:02 test-system {"_path":"ssl","_system_name":"test-system","_write_ts":"2026-05-11T17:22:02.759122Z","ts":"2026-05-11T17:22:02.754532Z","uid":"Cxu3mb4NZI2tAscrhb","id.orig_h":"10.205.140.1","id.orig_p":43108,"id.resp_h":"10.205.180.10","id.resp_p":443,"version":"TLSv12","cipher":"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","curve":"secp256r1","server_name":null,"resumed":false,"established":true,"ssl_history":"CsxknGIti","ja3":"c34a54599a1fbaf1786bb6d633545a60","ja3s":"03788d8896c247631984a250db971b74","subject":"CN=*.org.test.io,O=Test Org,L=Los Angeles,ST=California,C=US","issuer":"CN=Test Issuer,OU=Test OU,O=Test Org,C=US","validation_status":"ok","cert_chain_fps":["3f5332c4631559ebf68bacbdf27d408g5d904bf5fa8e4f455c5274aae330ca96","8bb2f6883fed289a521ba27c478482950874e167caccec6fc025990c0c46813f"],"port":23367} + tags: + - source:LOGS_SOURCE + timestamp: 1778520122754 + - + sample: "<134>May 11 17:22:02 test-system {\"_path\":\"ssl_red\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T17:22:02.032606Z\",\"ts\":\"2026-05-11T17:22:02.023967Z\",\"uid\":\"CfSalo2IDfDMZeOJ02\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":54745,\"id.resp_h\":\"18.164.5.1\",\"id.resp_p\":443,\"id.vlan\":1000,\"version\":\"TLSv13\",\"cipher\":\"TLS_AES_128_GCM_SHA256\",\"curve\":\"x25519\",\"server_name\":\"public.test.app\",\"resumed\":false,\"established\":true,\"ssl_history\":\"CsiI\",\"ja3\":\"f4febc55ea12b31ae17cfb7e614afea4\",\"ja3s\":\"f4febc55ea12b31ae17cfb7e614afea4\",\"port\":64884}" + service: "corelight" + result: + custom: + cipher: TLS_AES_128_GCM_SHA256 + established: true + server_name: public.test.app + curve: x25519 + ja3s: f4febc55ea12b31ae17cfb7e614afea4 + _write_ts: '2026-05-11T17:22:02.032606Z' + ssl_history: CsiI + _path: ssl_red + version: TLSv13 + network: + destination: + port: 443 + ip: 18.164.5.1 + client: + port: 54745 + ip: 192.168.1.1 + uid: CfSalo2IDfDMZeOJ02 + ocsf: + severity: Informational + activity_name: Open + metadata: + uid: CfSalo2IDfDMZeOJ02 + product: + name: Zeek + vendor_name: Corelight + original_time: '2026-05-11T17:22:02.023967Z' + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 54745 + ip: 192.168.1.1 + status_id: 1 + class_uid: 4001 + activity_id: 1 + tls: + cipher: TLS_AES_128_GCM_SHA256 + ja3s_hash: + value: f4febc55ea12b31ae17cfb7e614afea4 + algorithm_id: 1 + algorithm: MD5 + ja3_hash: + value: f4febc55ea12b31ae17cfb7e614afea4 + algorithm_id: 1 + algorithm: MD5 + version: TLSv13 + sni: public.test.app + time: 1778520122023 + dst_endpoint: + hostname: public.test.app + port: 443 + ip: 18.164.5.1 + severity_id: 1 + class_name: Network Activity + status: Success + _system_name: test-system + port: 64884 + id: + vlan: 1000 + orig_p: 54745 + resp_h: 18.164.5.1 + orig_h: 192.168.1.1 + resp_p: 443 + resumed: false + ja3: f4febc55ea12b31ae17cfb7e614afea4 + ts: '2026-05-11T17:22:02.023967Z' + service: ssl_red + message: <134>May 11 17:22:02 test-system {"_path":"ssl_red","_system_name":"test-system","_write_ts":"2026-05-11T17:22:02.032606Z","ts":"2026-05-11T17:22:02.023967Z","uid":"CfSalo2IDfDMZeOJ02","id.orig_h":"192.168.1.1","id.orig_p":54745,"id.resp_h":"18.164.5.1","id.resp_p":443,"id.vlan":1000,"version":"TLSv13","cipher":"TLS_AES_128_GCM_SHA256","curve":"x25519","server_name":"public.test.app","resumed":false,"established":true,"ssl_history":"CsiI","ja3":"f4febc55ea12b31ae17cfb7e614afea4","ja3s":"f4febc55ea12b31ae17cfb7e614afea4","port":64884} + tags: + - source:LOGS_SOURCE + timestamp: 1778520122023 + - + sample: "<134>May 11 16:11:53 test-system {\"_path\":\"weird_red\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T16:11:53.944496Z\",\"ts\":\"2026-05-11T16:11:53.944496Z\",\"uid\":\"CYThCC2DZNVcGgpiI2\",\"id.orig_h\":\"10.250.182.4\",\"id.orig_p\":58438,\"id.resp_h\":\"180.153.200.12\",\"id.resp_p\":53,\"id.vlan\":1600,\"name\":\"DNS_truncated_quest_too_short\",\"source\":\"DNS\",\"notice\":false,\"peer\":\"worker-04\"}" + service: "corelight" + result: + custom: + _write_ts: '2026-05-11T16:11:53.944496Z' + _path: weird_red + source: DNS + network: + destination: + port: 53 + ip: 180.153.200.12 + client: + port: 58438 + ip: 10.250.182.4 + uid: CYThCC2DZNVcGgpiI2 + ocsf: + severity: Informational + activity_name: Traffic + metadata: + uid: CYThCC2DZNVcGgpiI2 + product: + name: Zeek + vendor_name: Corelight + event_code: DNS_truncated_quest_too_short + version: 1.5.0 + category_uid: 4 + category_name: Network Activity + src_endpoint: + port: 58438 + ip: 10.250.182.4 + message: DNS_truncated_quest_too_short + status_detail: DNS_truncated_quest_too_short + connection_info: + uid: CYThCC2DZNVcGgpiI2 + direction_id: 0 + protocol_name: DNS + direction: Unknown + class_uid: 4001 + activity_id: 6 + time: 1778515913944 + dst_endpoint: + port: 53 + ip: 180.153.200.12 + severity_id: 1 + class_name: Network Activity + _system_name: test-system + peer: worker-04 + name: DNS_truncated_quest_too_short + id: + vlan: 1600 + orig_p: 58438 + resp_h: 180.153.200.12 + orig_h: 10.250.182.4 + resp_p: 53 + ts: '2026-05-11T16:11:53.944496Z' + notice: false + service: weird_red + message: <134>May 11 16:11:53 test-system {"_path":"weird_red","_system_name":"test-system","_write_ts":"2026-05-11T16:11:53.944496Z","ts":"2026-05-11T16:11:53.944496Z","uid":"CYThCC2DZNVcGgpiI2","id.orig_h":"10.250.182.4","id.orig_p":58438,"id.resp_h":"180.153.200.12","id.resp_p":53,"id.vlan":1600,"name":"DNS_truncated_quest_too_short","source":"DNS","notice":false,"peer":"worker-04"} + tags: + - source:LOGS_SOURCE + timestamp: 1778515913944 + - + sample: "<134>May 11 19:11:03 test-system {\"_path\":\"notice\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T19:11:03.521194Z\",\"ts\":\"2026-05-11T19:11:03.521194Z\",\"note\":\"ATTACK::Discovery\",\"msg\":\"Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins\",\"id.orig_h\":\"10.10.10.15\",\"id.orig_p\":54321,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":80,\"peer_descr\":\"manager\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600, \"uid\":\"12345678901234567890\", \"severity\":{\"name\":\"High\",\"id\":4}}" + service: "corelight" + result: + custom: + msg: Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins + suppress_for: 3600 + _write_ts: '2026-05-11T19:11:03.521194Z' + network: + client: + ip: 10.10.10.15 + port: 54321 + destination: + ip: 192.168.1.1 + port: 80 + zeek: + note: ATTACK::Discovery + _path: notice + peer_descr: manager + ocsf: + activity_name: Create + metadata: + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + uid: '12345678901234567890' + event_code: ATTACK::Discovery + is_alert: true + finding_info: + uid: '12345678901234567890' + title: ATTACK::Discovery + status_id: 1 + time: 1778526663521 + status: New + severity: High + severity_id: 4 + evidences: + - src_endpoint: + ip: 10.10.10.15 + port: 54321 + dst_endpoint: + ip: 192.168.1.1 + port: 80 + category_uid: 2 + category_name: Findings + class_uid: 2004 + activity_id: 1 + class_name: Detection Finding + _system_name: test-system + actions: + - Notice::ACTION_LOG + ts: '2026-05-11T19:11:03.521194Z' + note: ATTACK::Discovery + uid: '12345678901234567890' + severity: + name: High + id: 4 + id: + orig_p: 54321 + resp_h: 192.168.1.1 + orig_h: 10.10.10.15 + resp_p: 80 + service: notice + message: <134>May 11 19:11:03 test-system {"_path":"notice","_system_name":"test-system","_write_ts":"2026-05-11T19:11:03.521194Z","ts":"2026-05-11T19:11:03.521194Z","note":"ATTACK::Discovery","msg":"Detected activity from host 10.10.10.15, total attempts 9 within timeframe 5.0 mins","id.orig_h":"10.10.10.15","id.orig_p":54321,"id.resp_h":"192.168.1.1","id.resp_p":80,"peer_descr":"manager","actions":["Notice::ACTION_LOG"],"suppress_for":3600, "uid":"12345678901234567890", "severity":{"name":"High","id":4}} + tags: + - source:LOGS_SOURCE + timestamp: 1778526663521 + status: info + - + sample: "<134>May 11 17:59:59 test-system {\"_path\":\"suricata_corelight\",\"_system_name\":\"test-system\",\"_write_ts\":\"2026-05-11T17:59:59.359532Z\",\"ts\":\"2026-05-11T17:59:59.358114Z\",\"uid\":\"CN4bklsrGiWMsQhg3\",\"id.orig_h\":\"10.215.3.8\",\"id.orig_p\":65196,\"id.resp_h\":\"10.69.5.8\",\"id.resp_p\":80,\"id.vlan\":1123,\"proto\":\"tcp\",\"service\":\"http\",\"suri_id\":\"SOHaIDWJ5dBe\",\"flow_id\":2093130069465879,\"tx_id\":0,\"community_id\":\"1:Y4yC2kKf1Cm0QUkA/aBLSDxepcY=\",\"severity\":1,\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":2006380,\"rev\":17,\"signature\":\"ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted\",\"category\":\"Potential Corporate Privacy Violation\",\"severity\":1,\"metadata\":[\"confidence:Medium\",\"created_at:2010_07_30\",\"performance_impact:Significant\",\"signature_severity:Informational\",\"updated_at:2024_08_07\"]}}" + service: "corelight" + result: + custom: + alert_severity: High + _write_ts: '2026-05-11T17:59:59.359532Z' + suri_id: SOHaIDWJ5dBe + _path: suricata_corelight + tx_id: 0 + network: + destination: + port: 80 + ip: 10.69.5.8 + client: + port: 65196 + ip: 10.215.3.8 + uid: CN4bklsrGiWMsQhg3 + community_id: 1:Y4yC2kKf1Cm0QUkA/aBLSDxepcY= + ocsf: + severity: High + activity_name: Create + metadata: + uid: CN4bklsrGiWMsQhg3 + product: + name: Suricata + vendor_name: Corelight + event_code: '2006380' + log_provider: Corelight + version: 1.5.0 + category_uid: 2 + category_name: Findings + confidence: Medium + is_alert: true + status_detail: allowed + finding_info: + uid_alt: SOHaIDWJ5dBe + title: ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted + analytic: + uid: '2006380' + type_id: 1 + name: ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted + type: Rule + uid: CN4bklsrGiWMsQhg3 + class_uid: 2004 + activity_id: 1 + time: 1778522399358 + severity_id: 4 + evidences: + - src_endpoint: + port: 65196 + ip: 10.215.3.8 + dst_endpoint: + port: 80 + ip: 10.69.5.8 + class_name: Detection Finding + confidence_id: 2 + _system_name: test-system + alert: + severity: 1 + signature_id: 2006380 + rev: 17 + metadata: + - confidence:Medium + - created_at:2010_07_30 + - performance_impact:Significant + - signature_severity:Informational + - updated_at:2024_08_07 + gid: 1 + signature: ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted + action: allowed + category: Potential Corporate Privacy Violation + service: http + flow_id: 2093130069465879 + proto: tcp + id: + vlan: 1123 + orig_p: 65196 + resp_h: 10.69.5.8 + orig_h: 10.215.3.8 + resp_p: 80 + ts: '2026-05-11T17:59:59.358114Z' + severity: 1 + service: suricata_corelight + message: <134>May 11 17:59:59 test-system {"_path":"suricata_corelight","_system_name":"test-system","_write_ts":"2026-05-11T17:59:59.359532Z","ts":"2026-05-11T17:59:59.358114Z","uid":"CN4bklsrGiWMsQhg3","id.orig_h":"10.215.3.8","id.orig_p":65196,"id.resp_h":"10.69.5.8","id.resp_p":80,"id.vlan":1123,"proto":"tcp","service":"http","suri_id":"SOHaIDWJ5dBe","flow_id":2093130069465879,"tx_id":0,"community_id":"1:Y4yC2kKf1Cm0QUkA/aBLSDxepcY=","severity":1,"alert":{"action":"allowed","gid":1,"signature_id":2006380,"rev":17,"signature":"ET POLICY Outgoing Basic Auth Base64 HTTP Password detected unencrypted","category":"Potential Corporate Privacy Violation","severity":1,"metadata":["confidence:Medium","created_at:2010_07_30","performance_impact:Significant","signature_severity:Informational","updated_at:2024_08_07"]}} + tags: + - source:LOGS_SOURCE + timestamp: 1778522399358 + status: alert + - + sample: "<134>May 11 19:26:26 ndr-dub-stryker-DC-1 {\"_path\":\"files\",\"_system_name\":\"ndr-dub-stryker-DC-1\",\"_write_ts\":\"2026-05-11T19:26:26.082433Z\",\"ts\":\"2026-05-11T19:26:25.875206Z\",\"uid\":\"CjTuQU17IDvaVa8Nq2\",\"fuid\":\"FOPDsn3PdkiZsljcj2\",\"tx_hosts\":[\"10.104.10.60\"],\"rx_hosts\":[\"10.104.10.65\"],\"conn_uids\":[\"CjTuQU17IDvaGb8Nq2\"],\"source\":\"HTTP\",\"depth\":0,\"analyzers\":[\"SHA1\",\"MD5\",\"SHA256\",\"DATA_EVENT\"],\"local_orig\":true,\"is_orig\":false,\"seen_bytes\":253109,\"total_bytes\":253109,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"duration\":0.2072269916534424,\"mime_type\":\"text/json\",\"md5\":\"6e6ae0ed19f595687684faafae5499e13\",\"sha1\":\"f6578daa6d398c91398888b91a96d4c0e099c79c\",\"sha256\":\"a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f\",\"id.vlan\":1010}" + service: "corelight" + result: + custom: + timedout: false + sha256: a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f + source: HTTP + duration: 0.2072269916534424 + uid: CjTuQU17IDvaVa8Nq2 + analyzers: + - SHA1 + - MD5 + - SHA256 + - DATA_EVENT + ocsf: + severity: Informational + activity_name: Download + metadata: + uid: CjTuQU17IDvaVa8Nq2 + product: + name: Zeek + vendor_name: Corelight + version: 1.5.0 + category_uid: 6 + category_name: Application Activity + actor: + session: + uid: CjTuQU17IDvaVa8Nq2 + file: + uid: FOPDsn3PdkiZsljcj2 + size: 253109 + mime_type: text/json + type_id: 1 + name: FOPDsn3PdkiZsljcj2 + hashes: + - algorithm_id: 1 + value: 6e6ae0ed19f595687684faafae5499e13 + algorithm: MD5 + - algorithm_id: 2 + value: f6578daa6d398c91398888b91a96d4c0e099c79c + algorithm: SHA-1 + - algorithm_id: 3 + value: a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f + algorithm: SHA-256 + status_id: 1 + class_uid: 6006 + activity_id: 2 + time: 1778527585875 + severity_id: 1 + class_name: File Hosting Activity + status: Success + src_endpoint: + ip: 10.104.10.60 + dst_endpoint: + ip: 10.104.10.65 + fuid: FOPDsn3PdkiZsljcj2 + id: + vlan: 1010 + local_orig: true + _write_ts: '2026-05-11T19:26:26.082433Z' + rx_hosts: + - 10.104.10.65 + zeek: + total_bytes: 253109 + seen_bytes: 253109 + missing_bytes: 0 + overflow_bytes: 0 + orig_host: + - 10.104.10.65 + _path: files + is_orig: false + tx_hosts: + - 10.104.10.60 + sha1: f6578daa6d398c91398888b91a96d4c0e099c79c + depth: 0 + _system_name: ndr-dub-stryker-DC-1 + mime_type: text/json + dest_host: + - 10.104.10.60 + conn_uids: + - CjTuQU17IDvaGb8Nq2 + ts: '2026-05-11T19:26:25.875206Z' + md5: 6e6ae0ed19f595687684faafae5499e13 + total_bytes: 253109 + seen_bytes: 253109 + missing_bytes: 0 + service: files + message: <134>May 11 19:26:26 ndr-dub-stryker-DC-1 {"_path":"files","_system_name":"ndr-dub-stryker-DC-1","_write_ts":"2026-05-11T19:26:26.082433Z","ts":"2026-05-11T19:26:25.875206Z","uid":"CjTuQU17IDvaVa8Nq2","fuid":"FOPDsn3PdkiZsljcj2","tx_hosts":["10.104.10.60"],"rx_hosts":["10.104.10.65"],"conn_uids":["CjTuQU17IDvaGb8Nq2"],"source":"HTTP","depth":0,"analyzers":["SHA1","MD5","SHA256","DATA_EVENT"],"local_orig":true,"is_orig":false,"seen_bytes":253109,"total_bytes":253109,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"duration":0.2072269916534424,"mime_type":"text/json","md5":"6e6ae0ed19f595687684faafae5499e13","sha1":"f6578daa6d398c91398888b91a96d4c0e099c79c","sha256":"a7d5f44561e9707b3faf6ca1cdec4823e6625dd1c3aba2b7395697d65b47dc8f","id.vlan":1010} + tags: + - source:LOGS_SOURCE + timestamp: 1778527585875 \ No newline at end of file From 5ee80d2aea70a47ac4515d7b3b4bfeeaa8c086a1 Mon Sep 17 00:00:00 2001 From: jbfeldman-dd Date: Mon, 18 May 2026 14:54:58 -0400 Subject: [PATCH 4/4] Add OCSF DNS Activity normalization to coredns pipeline (#23726) * Add OCSF DNS Activity normalization to coredns pipeline Map CoreDNS query/response logs to OCSF DNS Activity [4003]. Adds OCSF facets, a single-class sub-pipeline (no pre-transformation), and the generated expected OCSF blocks in the test fixtures. Co-Authored-By: Claude Opus 4.7 (1M context) * Align coredns OCSF facet names with cloudflare and route53 validate-logs flagged five OCSF facet path conflicts. Rename to the canonical form used by the existing DNS integrations and add the `type: integer` annotation expected on `ocsf.rcode_id` and `ocsf.src_endpoint.port`. Co-Authored-By: Claude Opus 4.7 (1M context) * Add facetType range to ocsf.src_endpoint.port facet validate-logs asks for `facetType: range` on this facet path. Match the form CI's canonical-suggestion message printed for ocsf.src_endpoint.port. Co-Authored-By: Claude Opus 4.7 (1M context) * remove redundant fallbacks --------- Co-authored-by: Claude Opus 4.7 (1M context) --- coredns/assets/logs/coredns.yaml | 410 +++++++++++++++++++++++++ coredns/assets/logs/coredns_tests.yaml | 90 +++++- 2 files changed, 487 insertions(+), 13 deletions(-) diff --git a/coredns/assets/logs/coredns.yaml b/coredns/assets/logs/coredns.yaml index c1135c42f5447..5dd9552482c92 100644 --- a/coredns/assets/logs/coredns.yaml +++ b/coredns/assets/logs/coredns.yaml @@ -62,6 +62,99 @@ facets: name: DNSSEC path: dns.dnssec source: log + - groups: + - OCSF + name: Activity ID + path: ocsf.activity_id + source: log + - groups: + - OCSF + name: Activity Name + path: ocsf.activity_name + source: log + - groups: + - OCSF + name: Category ID + path: ocsf.category_uid + source: log + - groups: + - OCSF + name: Category + path: ocsf.category_name + source: log + - groups: + - OCSF + name: Class ID + path: ocsf.class_uid + source: log + - groups: + - OCSF + name: Class + path: ocsf.class_name + source: log + - groups: + - OCSF + name: Status ID + path: ocsf.status_id + source: log + - groups: + - OCSF + name: Status + path: ocsf.status + source: log + - groups: + - OCSF + name: Severity ID + path: ocsf.severity_id + source: log + - groups: + - OCSF + name: Severity + path: ocsf.severity + source: log + - groups: + - OCSF + name: Response Code ID + path: ocsf.rcode_id + source: log + type: integer + - groups: + - OCSF + name: Response Code + path: ocsf.rcode + source: log + - groups: + - OCSF + name: Source IP Address + path: ocsf.src_endpoint.ip + source: log + - facetType: range + groups: + - OCSF + name: Src Endpoint Port + path: ocsf.src_endpoint.port + source: log + type: integer + - groups: + - OCSF + name: DNS Query Hostname + path: ocsf.query.hostname + source: log + - groups: + - OCSF + name: Query Resource Record Type + path: ocsf.query.type + source: log + - groups: + - OCSF + name: Query Class + path: ocsf.query.class + source: log + - groups: + - OCSF + name: Protocol Name + path: ocsf.connection_info.protocol_name + source: log pipeline: type: pipeline name: CoreDNS @@ -90,3 +183,320 @@ pipeline: enabled: true sources: - level + - type: pipeline + name: OCSF sub pipeline for class DNS Activity [4003] + enabled: true + ocsf: + isOcsf: true + filter: + query: "@dns.question.name:*" + processors: + - type: string-builder-processor + name: Add ocsf.metadata.product.name + enabled: true + template: CoreDNS + target: ocsf.metadata.product.name + replaceMissing: false + - type: string-builder-processor + name: Add ocsf.metadata.product.vendor_name + enabled: true + template: CoreDNS + target: ocsf.metadata.product.vendor_name + replaceMissing: false + - type: arithmetic-processor + name: Convert `duration` (ns) to `ocsf.response_time` (ms) + enabled: true + expression: duration / 1000000 + target: ocsf.response_time + replaceMissing: false + - type: schema-processor + name: Apply OCSF schema for 4003 + enabled: true + schema: + schemaType: ocsf + version: 1.5.0 + className: DNS Activity + classUid: 4003 + extensions: [] + profiles: [] + mappers: + - name: ocsf.activity_id + categories: + - filter: + query: "*" + name: Traffic + id: 6 + targets: + name: ocsf.activity_name + id: ocsf.activity_id + type: schema-category-mapper + - name: ocsf.connection_info.direction_id + categories: + - filter: + query: "*" + name: Inbound + id: 1 + targets: + name: ocsf.connection_info.direction + id: ocsf.connection_info.direction_id + type: schema-category-mapper + - name: ocsf.severity_id + categories: + - filter: + query: "@level:(INFO OR DEBUG)" + name: Informational + id: 1 + - filter: + query: "@level:(WARN OR WARNING)" + name: Medium + id: 3 + - filter: + query: "@level:ERROR" + name: High + id: 4 + - filter: + query: "@level:FATAL" + name: Fatal + id: 6 + - filter: + query: "*" + name: Other + id: 99 + targets: + name: ocsf.severity + id: ocsf.severity_id + fallback: + values: + ocsf.severity: Other + ocsf.severity_id: "99" + sources: + ocsf.severity: + - level + type: schema-category-mapper + - name: ocsf.status_id + categories: + - filter: + query: "@dns.flags.rcode:NOERROR" + name: Success + id: 1 + - filter: + query: "*" + name: Failure + id: 2 + targets: + name: ocsf.status + id: ocsf.status_id + type: schema-category-mapper + - name: ocsf.rcode_id + categories: + - filter: + query: "@dns.flags.rcode:NOERROR" + name: NoError + id: 0 + - filter: + query: "@dns.flags.rcode:FORMERR" + name: FormError + id: 1 + - filter: + query: "@dns.flags.rcode:SERVFAIL" + name: ServError + id: 2 + - filter: + query: "@dns.flags.rcode:NXDOMAIN" + name: NXDomain + id: 3 + - filter: + query: "@dns.flags.rcode:(NOTIMP OR NOTIMPL)" + name: NotImp + id: 4 + - filter: + query: "@dns.flags.rcode:REFUSED" + name: Refused + id: 5 + - filter: + query: "@dns.flags.rcode:YXDOMAIN" + name: YXDomain + id: 6 + - filter: + query: "@dns.flags.rcode:YXRRSET" + name: YXRRSet + id: 7 + - filter: + query: "@dns.flags.rcode:NXRRSET" + name: NXRRSet + id: 8 + - filter: + query: "@dns.flags.rcode:NOTAUTH" + name: NotAuth + id: 9 + - filter: + query: "@dns.flags.rcode:NOTZONE" + name: NotZone + id: 10 + - filter: + query: "@dns.flags.rcode:(DSOTYPENI OR RCODE11)" + name: DSOTYPENI + id: 11 + - filter: + query: "@dns.flags.rcode:(BADSIG OR BADVERS)" + name: BADSIG_VERS + id: 16 + - filter: + query: "@dns.flags.rcode:BADKEY" + name: BADKEY + id: 17 + - filter: + query: "@dns.flags.rcode:BADTIME" + name: BADTIME + id: 18 + - filter: + query: "@dns.flags.rcode:BADMODE" + name: BADMODE + id: 19 + - filter: + query: "@dns.flags.rcode:BADNAME" + name: BADNAME + id: 20 + - filter: + query: "@dns.flags.rcode:BADALG" + name: BADALG + id: 21 + - filter: + query: "@dns.flags.rcode:BADTRUNC" + name: BADTRUNC + id: 22 + - filter: + query: "@dns.flags.rcode:BADCOOKIE" + name: BADCOOKIE + id: 23 + - filter: + query: "@dns.flags.rcode:(RCODE12 OR RCODE13 OR RCODE14 OR RCODE15)" + name: Unassigned + id: 24 + - filter: + query: "@dns.flags.rcode:RCODE65535" + name: Reserved + id: 25 + - filter: + query: "*" + name: Other + id: 99 + targets: + name: ocsf.rcode + id: ocsf.rcode_id + fallback: + values: + ocsf.rcode: Other + ocsf.rcode_id: "99" + sources: + ocsf.rcode: + - dns.flags.rcode + type: schema-category-mapper + - name: Map `ocsf.metadata.product.name` to `ocsf.metadata.product.name` + sources: + - ocsf.metadata.product.name + sourceType: attribute + target: ocsf.metadata.product.name + targetFormat: string + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `ocsf.metadata.product.vendor_name` to `ocsf.metadata.product.vendor_name` + sources: + - ocsf.metadata.product.vendor_name + sourceType: attribute + target: ocsf.metadata.product.vendor_name + targetFormat: string + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `ocsf.response_time` to `ocsf.response_time` + sources: + - ocsf.response_time + sourceType: attribute + target: ocsf.response_time + targetFormat: integer + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `level` to `ocsf.metadata.log_level` + sources: + - level + sourceType: attribute + target: ocsf.metadata.log_level + targetFormat: string + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `timestamp` to `ocsf.time` + sources: + - timestamp + sourceType: attribute + target: ocsf.time + targetFormat: integer + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `network.client.ip` to `ocsf.src_endpoint.ip` + sources: + - network.client.ip + sourceType: attribute + target: ocsf.src_endpoint.ip + targetFormat: string + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `network.client.port` to `ocsf.src_endpoint.port` + sources: + - network.client.port + sourceType: attribute + target: ocsf.src_endpoint.port + targetFormat: integer + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `dns.question.name` to `ocsf.query.hostname` + sources: + - dns.question.name + sourceType: attribute + target: ocsf.query.hostname + targetFormat: string + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `dns.question.type` to `ocsf.query.type` + sources: + - dns.question.type + sourceType: attribute + target: ocsf.query.type + targetFormat: string + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `dns.question.class` to `ocsf.query.class` + sources: + - dns.question.class + sourceType: attribute + target: ocsf.query.class + targetFormat: string + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `dns.protocol` to `ocsf.connection_info.protocol_name` + sources: + - dns.protocol + sourceType: attribute + target: ocsf.connection_info.protocol_name + targetFormat: string + preserveSource: true + overrideOnConflict: true + type: schema-remapper + - name: Map `dns.flags.rcode` to `ocsf.status_code` + sources: + - dns.flags.rcode + sourceType: attribute + target: ocsf.status_code + targetFormat: string + preserveSource: true + overrideOnConflict: true + type: schema-remapper diff --git a/coredns/assets/logs/coredns_tests.yaml b/coredns/assets/logs/coredns_tests.yaml index a77cbacd39062..9eb3610f85326 100644 --- a/coredns/assets/logs/coredns_tests.yaml +++ b/coredns/assets/logs/coredns_tests.yaml @@ -6,8 +6,8 @@ tests: custom: dns: answer: - size: 174.0 - buffer: 512.0 + size: 174 + buffer: 512 dnssec: "false" flags: list: @@ -16,21 +16,54 @@ tests: - "rd" - "ra" rcode: "NXDOMAIN" - id: 21628.0 + id: 21628 protocol: "udp" question: class: "IN" name: "trace-k8s.agent.datadoghq.com.apm1.cluster.local." - size: 66.0 + size: 66 type: "AAAA" - duration: 78493.0 + duration: 78493 level: "INFO" network: client: ip: "10.145.105.176" port: "36008" + ocsf: + activity_id: 6 + activity_name: "Traffic" + category_name: "Network Activity" + category_uid: 4 + class_name: "DNS Activity" + class_uid: 4003 + connection_info: + direction: "Inbound" + direction_id: 1 + protocol_name: "udp" + metadata: + log_level: "INFO" + product: + name: "CoreDNS" + vendor_name: "CoreDNS" + version: "1.5.0" + query: + class: "IN" + hostname: "trace-k8s.agent.datadoghq.com.apm1.cluster.local." + type: "AAAA" + rcode: "NXDomain" + rcode_id: 3 + response_time: 0 + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "10.145.105.176" + port: 36008 + status: "Failure" + status_code: "NXDOMAIN" + status_id: 2 + time: 1574952440550 timestamp: 1574952440550 - message: "2019-11-28T14:47:20.55Z [INFO] 10.145.105.176:36008 - 21628 \"AAAA IN trace-k8s.agent.datadoghq.com.apm1.cluster.local. udp 66 false 512\" NXDOMAIN qr,aa,rd,ra 174 0.000078493s" + message: '2019-11-28T14:47:20.55Z [INFO] 10.145.105.176:36008 - 21628 "AAAA IN trace-k8s.agent.datadoghq.com.apm1.cluster.local. udp 66 false 512" NXDOMAIN qr,aa,rd,ra 174 0.000078493s' status: "info" tags: - "source:LOGS_SOURCE" @@ -41,8 +74,8 @@ tests: custom: dns: answer: - size: 68.0 - buffer: 4096.0 + size: 68 + buffer: 4096 dnssec: "false" flags: list: @@ -51,22 +84,53 @@ tests: - "ra" - "ad" rcode: "NOERROR" - id: 29008.0 + id: 29008 protocol: "udp" question: class: "IN" name: "example.org." - size: 41.0 + size: 41 type: "A" - duration: 3.7990251E7 + duration: 37990251 level: "INFO" network: client: ip: "127.0.0.1" port: "50759" - message: "[INFO] 127.0.0.1:50759 - 29008 \"A IN example.org. udp 41 false 4096\" NOERROR qr,rd,ra,ad 68 0.037990251s" + ocsf: + activity_id: 6 + activity_name: "Traffic" + category_name: "Network Activity" + category_uid: 4 + class_name: "DNS Activity" + class_uid: 4003 + connection_info: + direction: "Inbound" + direction_id: 1 + protocol_name: "udp" + metadata: + log_level: "INFO" + product: + name: "CoreDNS" + vendor_name: "CoreDNS" + version: "1.5.0" + query: + class: "IN" + hostname: "example.org." + type: "A" + rcode: "NoError" + rcode_id: 0 + response_time: 37 + severity: "Informational" + severity_id: 1 + src_endpoint: + ip: "127.0.0.1" + port: 50759 + status: "Success" + status_code: "NOERROR" + status_id: 1 + message: '[INFO] 127.0.0.1:50759 - 29008 "A IN example.org. udp 41 false 4096" NOERROR qr,rd,ra,ad 68 0.037990251s' status: "info" tags: - "source:LOGS_SOURCE" timestamp: 1 -