Skip to content

Commit b4466fa

Browse files
mjabascal10mjabascal10
committed
feat(filter): add Linux filter update with enhanced JSON parsing and field normalization
1 parent 34d8fad commit b4466fa

File tree

2 files changed

+395
-0
lines changed

2 files changed

+395
-0
lines changed

‎backend/src/main/resources/config/liquibase/changelog/20260220001_update_filter_linux.xml‎

Lines changed: 393 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,393 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
6+
7+
<changeSet id="20260210007" author="Manuel">
8+
9+
<sql dbms="postgresql" splitStatements="true" stripComments="true">
10+
<![CDATA[
11+
12+
UPDATE public.utm_logstash_filter
13+
SET filter_version='4.0.0',
14+
updated_at=now(),
15+
logstash_filter=$$# System Linux filter version 4.0.0
16+
# Support for systemd/journald JSON format from filebeat/journald
17+
# Converts SCREAMINGSNAKECASE and snakecase to camelCase
18+
# Maps to UTMStack Standard Event Schema
19+
# Optimized: Direct mapping to standard schema (no intermediate steps)
20+
21+
pipeline:
22+
- dataTypes:
23+
- linux
24+
steps:
25+
# ========================================
26+
# PHASE 1: EXTRACTION
27+
# ========================================
28+
29+
# Parse JSON from systemd/journald
30+
- json:
31+
source: raw
32+
where: 'startsWith("raw", "{")'
33+
34+
# ========================================
35+
# PHASE 2: FIELD NORMALIZATION (camelCase conversion)
36+
# ========================================
37+
38+
# Convert SCREAMINGSNAKECASE to camelCase
39+
- rename:
40+
from:
41+
- log.MESSAGE
42+
to: log.message
43+
where: exists("log.MESSAGE")
44+
45+
- rename:
46+
from:
47+
- log.PRIORITY
48+
to: log.priority
49+
where: exists("log.PRIORITY")
50+
51+
- rename:
52+
from:
53+
- log.SYSLOGIDENTIFIER
54+
to: log.syslogIdentifier
55+
where: exists("log.SYSLOGIDENTIFIER")
56+
57+
- rename:
58+
from:
59+
- log.SYSLOGTIMESTAMP
60+
to: log.syslogTimestamp
61+
where: exists("log.SYSLOGTIMESTAMP")
62+
63+
- rename:
64+
from:
65+
- log.SYSLOGFACILITY
66+
to: log.syslogFacility
67+
where: exists("log.SYSLOGFACILITY")
68+
69+
- rename:
70+
from:
71+
- log.SYSLOGPID
72+
to: log.syslogPid
73+
where: exists("log.SYSLOGPID")
74+
75+
# Convert snakecase to camelCase (only for fields staying in log.*)
76+
- rename:
77+
from:
78+
- log.PID
79+
to: log.pid
80+
where: exists("log.PID")
81+
82+
- rename:
83+
from:
84+
- log.UID
85+
to: log.uid
86+
where: exists("log.UID")
87+
88+
- rename:
89+
from:
90+
- log.GID
91+
to: log.gid
92+
where: exists("log.GID")
93+
94+
- rename:
95+
from:
96+
- log.TID
97+
to: log.tid
98+
where: exists("log.TID")
99+
100+
- rename:
101+
from:
102+
- log.EXE
103+
to: log.exe
104+
where: exists("log.EXE")
105+
106+
- rename:
107+
from:
108+
- log.UNIT
109+
to: log.unit
110+
where: exists("log.UNIT")
111+
112+
- rename:
113+
from:
114+
- log.SYSTEMDUNIT
115+
to: log.systemdUnit
116+
where: exists("log.SYSTEMDUNIT")
117+
118+
- rename:
119+
from:
120+
- log.SYSTEMDSLICE
121+
to: log.systemdSlice
122+
where: exists("log.SYSTEMDSLICE")
123+
124+
- rename:
125+
from:
126+
- log.SYSTEMDUSERSLICE
127+
to: log.systemdUserSlice
128+
where: exists("log.SYSTEMDUSERSLICE")
129+
130+
- rename:
131+
from:
132+
- log.SYSTEMDSESSION
133+
to: log.systemdSession
134+
where: exists("log.SYSTEMDSESSION")
135+
136+
- rename:
137+
from:
138+
- log.SESSIONID
139+
to: log.sessionId
140+
where: exists("log.SESSIONID")
141+
142+
- rename:
143+
from:
144+
- log.LEADER
145+
to: log.leader
146+
where: exists("log.LEADER")
147+
148+
- rename:
149+
from:
150+
- log.SYSTEMDOWNERUID
151+
to: log.systemdOwnerUid
152+
where: exists("log.SYSTEMDOWNERUID")
153+
154+
- rename:
155+
from:
156+
- log.SYSTEMDCGROUP
157+
to: log.systemdCgroup
158+
where: exists("log.SYSTEMDCGROUP")
159+
160+
- rename:
161+
from:
162+
- log.BOOTID
163+
to: log.bootId
164+
where: exists("log.BOOTID")
165+
166+
- rename:
167+
from:
168+
- log.MACHINEID
169+
to: log.machineId
170+
where: exists("log.MACHINEID")
171+
172+
- rename:
173+
from:
174+
- log.TRANSPORT
175+
to: log.transport
176+
where: exists("log.TRANSPORT")
177+
178+
- rename:
179+
from:
180+
- log.SELINUXCONTEXT
181+
to: log.selinuxContext
182+
where: exists("log.SELINUXCONTEXT")
183+
184+
- rename:
185+
from:
186+
- log.AUDITSESSION
187+
to: log.auditSession
188+
where: exists("log.AUDITSESSION")
189+
190+
- rename:
191+
from:
192+
- log.AUDITLOGINUID
193+
to: log.auditLoginUid
194+
where: exists("log.AUDITLOGINUID")
195+
196+
- rename:
197+
from:
198+
- log.CAPEFFECTIVE
199+
to: log.capEffective
200+
where: exists("log.CAPEFFECTIVE")
201+
202+
- rename:
203+
from:
204+
- log.REALTIMETIMESTAMP
205+
to: log.realtimeTimestamp
206+
where: exists("log.REALTIMETIMESTAMP")
207+
208+
- rename:
209+
from:
210+
- log.SOURCEREALTIMETIMESTAMP
211+
to: log.sourceRealtimeTimestamp
212+
where: exists("log.SOURCEREALTIMETIMESTAMP")
213+
214+
- rename:
215+
from:
216+
- log.MONOTONICTIMESTAMP
217+
to: log.monotonicTimestamp
218+
where: exists("log.MONOTONICTIMESTAMP")
219+
220+
- rename:
221+
from:
222+
- log.CURSOR
223+
to: log.cursor
224+
where: exists("log.CURSOR")
225+
226+
- rename:
227+
from:
228+
- log.SEQNUM
229+
to: log.seqnum
230+
where: exists("log.SEQNUM")
231+
232+
- rename:
233+
from:
234+
- log.SEQNUMID
235+
to: log.seqnumId
236+
where: exists("log.SEQNUMID")
237+
238+
- rename:
239+
from:
240+
- log.RUNTIMESCOPE
241+
to: log.runtimeScope
242+
where: exists("log.RUNTIMESCOPE")
243+
244+
- rename:
245+
from:
246+
- log.STREAMID
247+
to: log.streamId
248+
where: exists("log.STREAMID")
249+
250+
- rename:
251+
from:
252+
- log.SYSTEMDINVOCATIONID
253+
to: log.systemdInvocationId
254+
where: exists("log.SYSTEMDINVOCATIONID")
255+
256+
- rename:
257+
from:
258+
- log.CODEFILE
259+
to: log.codeFile
260+
where: exists("log.CODEFILE")
261+
262+
- rename:
263+
from:
264+
- log.CODELINE
265+
to: log.codeLine
266+
where: exists("log.CODELINE")
267+
268+
- rename:
269+
from:
270+
- log.CODEFUNC
271+
to: log.codeFunc
272+
where: exists("log.CODEFUNC")
273+
274+
- rename:
275+
from:
276+
- log.INVOCATIONID
277+
to: log.invocationId
278+
where: exists("log.INVOCATIONID")
279+
280+
- rename:
281+
from:
282+
- log.JOBID
283+
to: log.jobId
284+
where: exists("log.JOBID")
285+
286+
- rename:
287+
from:
288+
- log.JOBRESULT
289+
to: actionResult
290+
where: exists("log.JOBRESULT")
291+
292+
- rename:
293+
from:
294+
- log.JOBTYPE
295+
to: log.jobType
296+
where: exists("log.JOBTYPE")
297+
298+
- rename:
299+
from:
300+
- log.MESSAGEID
301+
to: log.messageId
302+
where: exists("log.MESSAGEID")
303+
304+
# ========================================
305+
# PHASE 3: STANDARD SCHEMA MAPPING
306+
# ========================================
307+
308+
# Map directly to Standard Event Schema (no intermediate camelCase step)
309+
- rename:
310+
from:
311+
- log.HOSTNAME
312+
to: origin.host
313+
where: exists("log.HOSTNAME")
314+
315+
- rename:
316+
from:
317+
- log.USERID
318+
to: origin.user
319+
where: exists("log.USERID")
320+
321+
- rename:
322+
from:
323+
- log.COMM
324+
to: origin.process
325+
where: exists("log.COMM")
326+
327+
- rename:
328+
from:
329+
- log.CMDLINE
330+
to: origin.command
331+
where: exists("log.CMDLINE")
332+
333+
# Map syslog priority (0-7) to severity labels
334+
- add:
335+
function: string
336+
params:
337+
key: severity
338+
value: "emergency"
339+
where: 'equals("log.priority", "0")'
340+
341+
- add:
342+
function: string
343+
params:
344+
key: severity
345+
value: "alert"
346+
where: 'equals("log.priority", "1")'
347+
348+
- add:
349+
function: string
350+
params:
351+
key: severity
352+
value: "critical"
353+
where: 'equals("log.priority", "2")'
354+
355+
- add:
356+
function: string
357+
params:
358+
key: severity
359+
value: "error"
360+
where: 'equals("log.priority", "3")'
361+
362+
- add:
363+
function: string
364+
params:
365+
key: severity
366+
value: "warning"
367+
where: 'equals("log.priority", "4")'
368+
369+
- add:
370+
function: string
371+
params:
372+
key: severity
373+
value: "notice"
374+
where: 'equals("log.priority", "5")'
375+
376+
- add:
377+
function: string
378+
params:
379+
key: severity
380+
value: "info"
381+
where: 'equals("log.priority", "6")'
382+
383+
- add:
384+
function: string
385+
params:
386+
key: severity
387+
value: "debug"
388+
where: 'equals("log.priority", "7")'$$
389+
WHERE id = 1413;
390+
]]>
391+
</sql>
392+
</changeSet>
393+
</databaseChangeLog>

0 commit comments

Comments
 (0)