Skip to content

Commit 2eb71ee

Browse files
mjabascal10mjabascal10
committed
feat(pfsense): update pfsense filter
1 parent fd8449c commit 2eb71ee

File tree

2 files changed

+374
-0
lines changed

2 files changed

+374
-0
lines changed

‎backend/src/main/resources/config/liquibase/changelog/20260129002_update_filter_pfsense.xml‎

Lines changed: 372 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,372 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
6+
7+
<changeSet id="20260129001" author="Manuel">
8+
9+
<sql dbms="postgresql" splitStatements="true" stripComments="true">
10+
<![CDATA[
11+
12+
UPDATE public.utm_logstash_filter
13+
SET filter_version='3.0.2',
14+
logstash_filter = $$ # PfSense Firewall filter, version 3.0.2
15+
# Based on docs and samples log provided
16+
# Support CSV format format
17+
#
18+
# Documentations
19+
# 1- https://docs.netgate.com/pfsense/en/latest/monitoring/logs/raw-filter-format.html
20+
#
21+
# Implementation
22+
# 1. Parsing the RAW field containing the PfSense
23+
# 2. Parsing headers of syslog the message
24+
# 3. Parsing the CSV field containing the PfSense
25+
26+
pipeline:
27+
- dataTypes:
28+
- firewall-pfsense
29+
steps:
30+
#......................................................................#
31+
#......................................................................#
32+
# Using grok to parse syslogHeader of the message
33+
#......................................................................#
34+
- grok:
35+
patterns:
36+
- fieldName: log.priority
37+
pattern: '\<{{.integer}}\>'
38+
- fieldName: log.syslogVersion
39+
pattern: '{{.integer}}'
40+
- fieldName: log.deviceTime
41+
pattern: '{{.year}}-{{.monthNumber}}-{{.monthDay}}{{.space}}{{.time}}{{.iso8601Timezone}}'
42+
- fieldName: log.syslogHost
43+
pattern: '{{.hostname}}'
44+
- fieldName: log.msgAll
45+
pattern: '{{.greedy}}'
46+
source: raw
47+
where: regexMatch("raw", "\\d{4}-\\d{2}-\\d{2}")
48+
49+
# Parsing syslog format date (OPNsense/pfSense)
50+
- grok:
51+
patterns:
52+
- fieldName: log.priority
53+
pattern: '\<{{.integer}}\>'
54+
- fieldName: log.deviceTime
55+
pattern: '{{.monthName}}{{.space}}{{.monthDay}}{{.space}}{{.time}}{{.space}}'
56+
- fieldName: log.syslogHost
57+
pattern: '{{.hostname}}{{.space}}'
58+
- fieldName: log.msgAll
59+
pattern: '{{.greedy}}'
60+
source: raw
61+
where: regexMatch("raw", "<\\d+>[A-Z][a-z]{2}\\s+\\d{1,2}\\s+\\d{2}")
62+
63+
#......................................................................#
64+
# Removing unnecessary characters of the syslogHeader
65+
#......................................................................#
66+
- trim:
67+
function: prefix
68+
substring: "<"
69+
fields:
70+
- log.priority
71+
- trim:
72+
function: suffix
73+
substring: ">"
74+
fields:
75+
- log.priority
76+
77+
#......................................................................#
78+
# Checking that the msgAll field exists
79+
#......................................................................#
80+
- grok:
81+
patterns:
82+
- fieldName: log.eventType
83+
pattern: '{{.word}}'
84+
- fieldName: log.pid
85+
pattern: '(\[)?({{.integer}}?)(\])?(- -|:)'
86+
- fieldName: log.csvMsg
87+
pattern: '{{.greedy}}'
88+
source: log.msgAll
89+
90+
#......................................................................#
91+
# Removing unnecessary characters
92+
#......................................................................#
93+
- trim:
94+
function: prefix
95+
substring: "["
96+
fields:
97+
- log.pid
98+
- trim:
99+
function: suffix
100+
substring: "]:"
101+
fields:
102+
- log.pid
103+
104+
# ..........................................................................#
105+
# Remove issues fileds
106+
# ..........................................................................#
107+
- delete:
108+
fields:
109+
- log.msgAll
110+
111+
#......................................................................#
112+
# Using csv to parse the message
113+
#......................................................................#
114+
- csv:
115+
source: log.csvMsg
116+
separator: ","
117+
headers:
118+
- log.ruleNumber
119+
- log.subRuleNumber
120+
- log.anchor
121+
- log.tracker
122+
- log.realInterface
123+
- log.reason
124+
- log.action
125+
- log.direction
126+
- log.ipVersion
127+
- log.ipv4Tos
128+
- log.ipv4Ecn
129+
- log.ipv4Ttl
130+
- log.ipv4Id
131+
- log.ipv4Offset
132+
- log.ipv4Flags
133+
- log.ipv4ProtocolId
134+
- log.proto
135+
- log.ipLength
136+
- log.srcIp
137+
- log.dstIp
138+
- log.srcPort
139+
- log.dstPort
140+
- log.dataLength
141+
- log.tcpFlags
142+
- log.sequenceNumber
143+
- log.ackNumber
144+
- log.tcpWindow
145+
- log.urg
146+
- log.tcpOptions
147+
where: regexMatch("log.csvMsg", "(.+),(\\s)?(match|\\w+),(block|pass),(in|out),(4|6),(.+)(tcp|TCP|Tcp)")
148+
149+
# .......................................................................#
150+
- csv:
151+
source: log.csvMsg
152+
separator: ","
153+
headers:
154+
- log.ruleNumber
155+
- log.subRuleNumber
156+
- log.anchor
157+
- log.tracker
158+
- log.realInterface
159+
- log.reason
160+
- log.action
161+
- log.direction
162+
- log.ipVersion
163+
- log.ipv4Tos
164+
- log.ipv4Ecn
165+
- log.ipv4Ttl
166+
- log.ipv4Id
167+
- log.ipv4Offset
168+
- log.ipv4Flags
169+
- log.ipv4ProtocolId
170+
- log.proto
171+
- log.ipLength
172+
- log.srcIp
173+
- log.dstIp
174+
- log.srcPort
175+
- log.dstPort
176+
- log.dataLength
177+
where: regexMatch("log.csvMsg", "(.+),(\\s)?(match|\\w+),(block|pass),(in|out),(4|6),(.+)(udp|UDP|Udp)")
178+
179+
#......................................................................#
180+
- csv:
181+
source: log.csvMsg
182+
separator: ","
183+
headers:
184+
- log.ruleNumber
185+
- log.subRuleNumber
186+
- log.anchor
187+
- log.tracker
188+
- log.realInterface
189+
- log.reason
190+
- log.action
191+
- log.direction
192+
- log.ipVersion
193+
- log.ipv4Tos
194+
- log.ipv4Ecn
195+
- log.ipv4Ttl
196+
- log.ipv4Id
197+
- log.ipv4Offset
198+
- log.ipv4Flags
199+
- log.ipv4ProtocolId
200+
- log.proto
201+
- log.ipLength
202+
- log.srcIp
203+
- log.dstIp
204+
- log.icmpType
205+
- log.icmpData1
206+
- log.icmpData2
207+
- log.icmpData3
208+
- log.icmpData4
209+
- log.icmpData5
210+
where: regexMatch("log.csvMsg", "(.+),(\\s)?(match|\\w+),(block|pass),(in|out),(4|6),(.+)(icmp|ICMP|Icmp)")
211+
212+
#......................................................................#
213+
- csv:
214+
source: log.csvMsg
215+
separator: ","
216+
headers:
217+
- log.ruleNumber
218+
- log.subRuleNumber
219+
- log.anchor
220+
- log.tracker
221+
- log.realInterface
222+
- log.reason
223+
- log.action
224+
- log.direction
225+
- log.ipVersion
226+
- log.ipv6Class
227+
- log.ipv6FlowLabel
228+
- log.ipv6HopLimit
229+
- log.proto
230+
- log.ipv6ProtocolId
231+
- log.ipLength
232+
- log.srcIp
233+
- log.dstIp
234+
- log.srcPort
235+
- log.dstPort
236+
- log.dataLength
237+
- log.tcpFlags
238+
- log.sequenceNumber
239+
- log.ackNumber
240+
- log.tcpWindow
241+
- log.urg
242+
- log.tcpOptions
243+
where: regexMatch("log.csvMsg", "(.+),(\\s)?(match|\\w+),(block|pass),(in|out),(6|17),(.+)(tcp|TCP|Tcp)")
244+
245+
#......................................................................#
246+
- csv:
247+
source: log.csvMsg
248+
separator: ","
249+
headers:
250+
- log.ruleNumber
251+
- log.subRuleNumber
252+
- log.anchor
253+
- log.tracker
254+
- log.realInterface
255+
- log.reason
256+
- log.action
257+
- log.direction
258+
- log.ipVersion
259+
- log.ipv6Class
260+
- log.ipv6FlowLabel
261+
- log.ipv6HopLimit
262+
- log.proto
263+
- log.ipv6ProtocolId
264+
- log.ipLength
265+
- log.srcIp
266+
- log.dstIp
267+
- log.srcPort
268+
- log.dstPort
269+
- log.dataLength
270+
where: regexMatch("log.csvMsg", "(.+),(match|\\w+),(block|pass),(in|out),6,(.+)(udp|UDP|Udp)")
271+
272+
#......................................................................#
273+
- csv:
274+
source: log.csvMsg
275+
separator: ","
276+
headers:
277+
- log.ruleNumber
278+
- log.subRuleNumber
279+
- log.anchor
280+
- log.tracker
281+
- log.realInterface
282+
- log.reason
283+
- log.action
284+
- log.direction
285+
- log.ipVersion
286+
- log.ipv6Class
287+
- log.ipv6FlowLabel
288+
- log.ipv6HopLimit
289+
- log.proto
290+
- log.ipv6ProtocolId
291+
- log.ipLength
292+
- log.srcIp
293+
- log.dstIp
294+
- log.icmpType
295+
- log.icmpData1
296+
- log.icmpData2
297+
- log.icmpData3
298+
- log.icmpData4
299+
- log.icmpData5
300+
where: regexMatch("log.csvMsg", "(.+),(match|\\w+),(block|pass),(in|out),(6|17),(.+)(icmp|ICMP|Icmp)")
301+
302+
# ................................................#
303+
# Rename fields
304+
# ................................................#
305+
- rename:
306+
from:
307+
- log.action
308+
to: action
309+
310+
- rename:
311+
from:
312+
- log.proto
313+
to: protocol
314+
315+
- rename:
316+
from:
317+
- log.srcIp
318+
to: origin.ip
319+
320+
- rename:
321+
from:
322+
- log.dstIp
323+
to: target.ip
324+
325+
- rename:
326+
from:
327+
- log.srcPort
328+
to: origin.port
329+
330+
- rename:
331+
from:
332+
- log.dstPort
333+
to: target.port
334+
335+
# ................................................#
336+
# Fileds conversions
337+
# ................................................#
338+
- cast:
339+
fields:
340+
- origin.port
341+
- target.port
342+
to: int
343+
344+
# Adding geolocation to origin.ip
345+
- dynamic:
346+
plugin: com.utmstack.geolocation
347+
params:
348+
source: origin.ip
349+
destination: origin.geolocation
350+
where: exists("origin.ip")
351+
352+
# Adding geolocation to target.ip
353+
- dynamic:
354+
plugin: com.utmstack.geolocation
355+
params:
356+
source: target.ip
357+
destination: target.geolocation
358+
where: exists("target.ip")
359+
360+
# ..........................................................................#
361+
# Remove issues fileds
362+
# ..........................................................................#
363+
- delete:
364+
fields:
365+
- log.csvMsg
366+
367+
$$
368+
WHERE id=1522;
369+
]]>
370+
</sql>
371+
</changeSet>
372+
</databaseChangeLog>

‎backend/src/main/resources/config/liquibase/master.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,8 @@
327327

328328
<include file="/config/liquibase/changelog/20260129001_update_filter_sophos_xg.xml" relativeToChangelogFile="false"/>
329329

330+
<include file="/config/liquibase/changelog/20260129002_update_filter_pfsense.xml" relativeToChangelogFile="false"/>
331+
330332

331333

332334
</databaseChangeLog>

0 commit comments

Comments
 (0)