Skip to content

Commit 49872b0

Browse files
mjabascal10mjabascal10
committed
feat: update Netflow filter update and modify Bit Defender filter configuration
1 parent 326c4da commit 49872b0

File tree

4 files changed

+300
-2
lines changed

4 files changed

+300
-2
lines changed

‎backend/src/main/resources/config/liquibase/changelog/20260211001_update_filter_bit_defender.xml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ pipeline:
223223
- log.productVersion
224224
- log.signatureID
225225
- log.eventType
226-
- log.severityLabel
226+
- log.severity
227227
228228
- trim:
229229
function: prefix
@@ -268,6 +268,7 @@ pipeline:
268268
fields:
269269
- log.0trash
270270
- log.1trash
271+
- log.2trash
271272
- log.restData
272273
- log.irrelevant
273274
- log.spt
@@ -277,6 +278,7 @@ pipeline:
277278
- log.dvc
278279
- log.request
279280
- log.dvcToParse
281+
- log.cefVersion
280282
$$
281283
WHERE id = 1514;
282284
]]>

‎backend/src/main/resources/config/liquibase/changelog/20260211002_update_utm_correlation_seq.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<sql dbms="postgresql" splitStatements="true" stripComments="true">
1010
<![CDATA[
1111
12-
ALTER SEQUENCE utm_correlation_rules_id_seq RESTART WITH 878;
12+
ALTER SEQUENCE utm_correlation_rules_id_seq RESTART WITH 879;
1313
]]>
1414
</sql>
1515
</changeSet>

‎backend/src/main/resources/config/liquibase/changelog/20260211003_update_filter_netflow.xml‎

Lines changed: 294 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,294 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
6+
7+
<changeSet id="20260211003" author="Manuel">
8+
9+
<sql dbms="postgresql" splitStatements="true" stripComments="true">
10+
<![CDATA[
11+
12+
UPDATE public.utm_logstash_filter
13+
SET filter_version='3.1.0',
14+
updated_at = now(),
15+
logstash_filter = $$ # Netflow firewall module filter, version 3.1.0
16+
# Based in docs and Netflow Generator (Solarwinds) for send log
17+
#
18+
# Documentations
19+
# 1- https://www.cisco.com/en/US/technologies/tk648/tk362/technologies_white_paper09186a00800a3db9.html
20+
# 2- http://www.iana.org/assignments/ipfix/ipfix.xhtml
21+
# 3- https://helpdesk.kaseya.com/hc/en-gb/articles/115003522631-How-to-view-NetFlow-in-WireShark
22+
# 4- https://www.solarwinds.com/free-tools/
23+
#
24+
# Implementation
25+
# 1. Parsing the RAW field containing the Netflow
26+
pipeline:
27+
- dataTypes:
28+
- netflow
29+
steps:
30+
31+
# Using the kv filter with default config, usefull in key-value logs
32+
- kv:
33+
fieldSplit: " "
34+
valueSplit: "="
35+
36+
# Remove fields that have issues with kv filter
37+
- delete:
38+
fields:
39+
- log.msg
40+
41+
# Using grok to parse kv issued fields
42+
- grok:
43+
patterns:
44+
- fieldName: log.irrelevant
45+
pattern: '{{.data}}(msg=)'
46+
- fieldName: log.msg
47+
pattern: '{{.data}}({{.word}}=)'
48+
- fieldName: log.irrelevant
49+
pattern: '{{.greedy}}'
50+
source: log.kvMessage
51+
52+
# Using grok to remove irrelevant data
53+
- grok:
54+
patterns:
55+
- fieldName: log.msg
56+
pattern: '{{.greedy}}{{.space}}'
57+
- fieldName: log.irrelevant
58+
pattern: '{{.word}}(=)'
59+
source: log.msg
60+
61+
# Using grok to parse fields that have issues with kv filter
62+
- grok:
63+
patterns:
64+
- fieldName: log.irrelevant
65+
pattern: '{{.data}}bytes="['
66+
- fieldName: log.totalBytes
67+
pattern: '{{.data}}]"'
68+
source: raw
69+
70+
- grok:
71+
patterns:
72+
- fieldName: log.irrelevant1
73+
pattern: '{{.data}}packets="['
74+
- fieldName: log.totalPackets
75+
pattern: '{{.data}}]"'
76+
source: raw
77+
78+
- grok:
79+
patterns:
80+
- fieldName: log.irrelevant2
81+
pattern: '{{.data}}inEth="['
82+
- fieldName: log.inEthernet
83+
pattern: '{{.data}}]"'
84+
source: raw
85+
86+
- grok:
87+
patterns:
88+
- fieldName: log.irrelevant3
89+
pattern: '{{.data}}outEth="['
90+
- fieldName: log.outEthernet
91+
pattern: '{{.data}}]"'
92+
source: raw
93+
94+
- grok:
95+
patterns:
96+
- fieldName: log.irrelevant4
97+
pattern: '{{.data}}proto="/['
98+
- fieldName: protocol
99+
pattern: '{{.data}}]"'
100+
source: raw
101+
102+
- grok:
103+
patterns:
104+
- fieldName: log.irrelevant5
105+
pattern: '{{.data}}srcPort="['
106+
- fieldName: log.srcPortg
107+
pattern: '{{.data}}]"'
108+
source: raw
109+
110+
- grok:
111+
patterns:
112+
- fieldName: log.irrelevant6
113+
pattern: '{{.data}}dstPort="['
114+
- fieldName: log.dstPortg
115+
pattern: '{{.data}}]"'
116+
source: raw
117+
118+
- grok:
119+
patterns:
120+
- fieldName: log.irrelevant7
121+
pattern: '{{.data}}dstMask="['
122+
- fieldName: log.destMask
123+
pattern: '{{.data}}]"'
124+
source: raw
125+
126+
- grok:
127+
patterns:
128+
- fieldName: log.irrelevant8
129+
pattern: '{{.data}}srcMask="['
130+
- fieldName: log.sourceMask
131+
pattern: '{{.data}}]"'
132+
source: raw
133+
134+
- grok:
135+
patterns:
136+
- fieldName: log.irrelevant9
137+
pattern: '{{.data}}tcpFlags="['
138+
- fieldName: log.tcpFlgs
139+
pattern: '{{.data}}]"'
140+
source: raw
141+
142+
- grok:
143+
patterns:
144+
- fieldName: log.irrelevant10
145+
pattern: '{{.data}}dstAs="['
146+
- fieldName: log.destAs
147+
pattern: '{{.data}}]"'
148+
source: raw
149+
150+
- grok:
151+
patterns:
152+
- fieldName: log.irrelevant11
153+
pattern: '{{.data}}srcAs="['
154+
- fieldName: log.sourceAs
155+
pattern: '{{.data}}]"'
156+
source: raw
157+
158+
# Rename filds
159+
- rename:
160+
from:
161+
- log.srcIp
162+
to: origin.ip
163+
- rename:
164+
from:
165+
- log.dstIp
166+
to: target.ip
167+
- rename:
168+
from:
169+
- log.src_ip
170+
to: origin.ip
171+
- rename:
172+
from:
173+
- log.dest_ip
174+
to: target.ip
175+
- rename:
176+
from:
177+
- log.srcPortg
178+
to: origin.port
179+
- rename:
180+
from:
181+
- log.src_port
182+
to: origin.port
183+
- rename:
184+
from:
185+
- log.dstPortg
186+
to: target.port
187+
- rename:
188+
from:
189+
- log.dest_port
190+
to: target.port
191+
192+
# Fields conversions
193+
- cast:
194+
to: 'int'
195+
fields:
196+
- origin.port
197+
- target.port
198+
199+
# Removing unused caracters
200+
- trim:
201+
function: prefix
202+
substring: '"'
203+
fields:
204+
- log.bytesIn
205+
- log.bytesOut
206+
- log.exporter
207+
- log.first
208+
- log.last
209+
- log.nextHop
210+
- log.packetsIn
211+
- log.packetsOut
212+
- log.version
213+
- target.ip
214+
- origin.ip
215+
- trim:
216+
function: suffix
217+
substring: '"'
218+
fields:
219+
- log.bytesIn
220+
- log.bytesOut
221+
- log.exporter
222+
- log.first
223+
- log.last
224+
- log.nextHop
225+
- log.packetsIn
226+
- log.packetsOut
227+
- log.version
228+
- target.ip
229+
- origin.ip
230+
231+
- trim:
232+
function: suffix
233+
substring: ']"'
234+
fields:
235+
- log.totalBytes
236+
- log.totalPackets
237+
- log.inEthernet
238+
- log.outEthernet
239+
- protocol
240+
- origin.port
241+
- target.port
242+
- log.destMask
243+
- log.sourceMask
244+
- log.tcpFlgs
245+
- log.destAs
246+
- log.sourceAs
247+
248+
# Adding geolocation to origin.ip
249+
- dynamic:
250+
plugin: com.utmstack.geolocation
251+
params:
252+
source: origin.ip
253+
destination: origin.geolocation
254+
where: exists("origin.ip")
255+
256+
# Adding geolocation to target.ip
257+
- dynamic:
258+
plugin: com.utmstack.geolocation
259+
params:
260+
source: target.ip
261+
destination: target.geolocation
262+
where: exists("target.ip")
263+
264+
# Removing unused fields
265+
- delete:
266+
fields:
267+
- log.bytes
268+
- log.packets
269+
- log.inEth
270+
- log.outEth
271+
- log.srcPort
272+
- log.dstMask
273+
- log.srcMask
274+
- log.tcpFlags
275+
- log.dstAs
276+
- log.srcAs
277+
- log.irrelevant
278+
- log.irrelevant1
279+
- log.irrelevant2
280+
- log.irrelevant3
281+
- log.irrelevant4
282+
- log.irrelevant5
283+
- log.irrelevant6
284+
- log.irrelevant7
285+
- log.irrelevant8
286+
- log.irrelevant9
287+
- log.irrelevant10
288+
- log.irrelevant11
289+
$$
290+
WHERE id = 1523;
291+
]]>
292+
</sql>
293+
</changeSet>
294+
</databaseChangeLog>

‎backend/src/main/resources/config/liquibase/master.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,8 @@
409409

410410
<include file="/config/liquibase/changelog/20260211002_update_utm_correlation_seq.xml" relativeToChangelogFile="false"/>
411411

412+
<include file="/config/liquibase/changelog/20260211003_update_filter_netflow.xml" relativeToChangelogFile="false"/>
413+
412414

413415

414416
</databaseChangeLog>

0 commit comments

Comments
 (0)