Skip to content

Commit f78177d

Browse files
nvandammenvandamme
authored
daemon: rule lists operators caching + lists matching enhancements (#1567)
* daemon-go: port layered list caching and matching strategies * daemon-go: use immutable rule/list snapshots and add load benchmarks * daemon-go: dropped gobwas, rule/list now use go filepath.Match for globbing
1 parent 32df33c commit f78177d

5 files changed

Lines changed: 831 additions & 30 deletions

File tree

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
*.pyc
33
*.profile
44

5+
ui/opensnitch/proto/
6+
daemon/ui/protocol/
7+
58
.vscode/
69
.idea/
710
.DS_Store

‎daemon/rule/loader.go‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"sort"
1111
"strings"
1212
"sync"
13+
"sync/atomic"
1314
"time"
1415

1516
"github.com/evilsocket/opensnitch/daemon/conman"
@@ -26,15 +27,20 @@ type Loader struct {
2627
watcher *fsnotify.Watcher
2728
rules map[string]*Rule
2829
activeRules []string
30+
activeSnapshot atomic.Pointer[activeRulesSnapshot]
2931
Path string
3032
liveReload bool
3133
liveReloadRunning bool
32-
checkSums bool
34+
checkSums atomic.Bool
3335
stopLiveReload chan struct{}
3436

3537
sync.RWMutex
3638
}
3739

40+
type activeRulesSnapshot struct {
41+
rules []*Rule
42+
}
43+
3844
// NewLoader loads rules from disk, and watches for changes made to the rules files
3945
// on disk.
4046
func NewLoader(liveReload bool) (*Loader, error) {
@@ -69,7 +75,7 @@ func (l *Loader) GetAll() map[string]*Rule {
6975
// EnableChecksums enables checksums field for rules globally.
7076
func (l *Loader) EnableChecksums(enable bool) {
7177
log.Debug("[rules loader] EnableChecksums: %v", enable)
72-
l.checkSums = enable
78+
l.checkSums.Store(enable)
7379
procmon.EventsCache.SetComputeChecksums(enable)
7480
procmon.EventsCache.AddChecksumHash(string(OpProcessHashMD5))
7581
}
@@ -113,6 +119,7 @@ func (l *Loader) Reload(path string) error {
113119
l.Lock()
114120
l.activeRules = make([]string, 0)
115121
l.rules = make(map[string]*Rule)
122+
l.activeSnapshot.Store(nil)
116123
l.Unlock()
117124
return l.Load(path)
118125
}
@@ -367,6 +374,7 @@ func (l *Loader) unmarshalOperatorList(op *Operator) error {
367374

368375
func (l *Loader) sortRules() {
369376
l.activeRules = make([]string, 0, len(l.rules))
377+
orderedRules := make([]*Rule, 0, len(l.rules))
370378
for k, r := range l.rules {
371379
// exclude not enabled rules from the list of active rules
372380
if !r.Enabled {
@@ -375,6 +383,10 @@ func (l *Loader) sortRules() {
375383
l.activeRules = append(l.activeRules, k)
376384
}
377385
sort.Strings(l.activeRules)
386+
for _, name := range l.activeRules {
387+
orderedRules = append(orderedRules, l.rules[name])
388+
}
389+
l.activeSnapshot.Store(&activeRulesSnapshot{rules: orderedRules})
378390
}
379391

380392
func (l *Loader) addUserRule(rule *Rule) {
@@ -495,12 +507,14 @@ Exit:
495507

496508
// FindFirstMatch will try match the connection against the existing rule set.
497509
func (l *Loader) FindFirstMatch(con *conman.Connection) (match *Rule) {
498-
l.RLock()
499-
defer l.RUnlock()
510+
snapshot := l.activeSnapshot.Load()
511+
if snapshot == nil {
512+
return nil
513+
}
514+
hasChecksums := l.checkSums.Load()
500515

501-
for _, idx := range l.activeRules {
502-
rule, _ := l.rules[idx]
503-
if rule.Match(con, l.checkSums) {
516+
for _, rule := range snapshot.rules {
517+
if rule.Match(con, hasChecksums) {
504518
// We have a match.
505519
// Save the rule in order to don't ask the user to take action,
506520
// and keep iterating until a Deny or a Priority rule appears.

‎daemon/rule/operator.go‎

Lines changed: 101 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"strconv"
99
"strings"
1010
"sync"
11+
"sync/atomic"
1112

1213
"github.com/evilsocket/opensnitch/daemon/conman"
1314
"github.com/evilsocket/opensnitch/daemon/core"
@@ -75,13 +76,32 @@ const (
7576
type opCallback func(value string) bool
7677
type opGenericCallback func(value interface{}) bool
7778

79+
type listRegexEntry struct {
80+
file string
81+
re *regexp.Regexp
82+
}
83+
84+
type listCacheSnapshot struct {
85+
lists map[string]interface{}
86+
domainWildcards domainWildcardTrie
87+
domainGlobs []string
88+
listExact map[string]struct{}
89+
listNets []*net.IPNet
90+
regexEntries []listRegexEntry
91+
}
92+
7893
// Operator represents what we want to filter of a connection, and how.
7994
type Operator struct {
8095
cb opCallback
8196
cbGeneric opGenericCallback
8297
re *regexp.Regexp
8398
netMask *net.IPNet
8499
lists map[string]interface{}
100+
domainWildcards domainWildcardTrie
101+
domainGlobs []string
102+
listExact map[string]struct{}
103+
listNets []*net.IPNet
104+
listSnapshot atomic.Pointer[listCacheSnapshot]
85105
exitMonitorChan chan (struct{})
86106
rangeMin uint64
87107
rangeMax uint64
@@ -178,10 +198,10 @@ func (o *Operator) Compile() error {
178198
o.cb = o.reListCmp
179199
} else if o.Operand == OpIPLists {
180200
o.loadLists()
181-
o.cb = o.simpleListsCmp
201+
o.cbGeneric = o.ipListsCmp
182202
} else if o.Operand == OpNetLists {
183203
o.loadLists()
184-
o.cbGeneric = o.ipNetCmp
204+
o.cbGeneric = o.netListsCmp
185205
} else if o.Operand == OpHashMD5Lists {
186206
o.loadLists()
187207
o.cb = o.simpleListsCmp
@@ -290,9 +310,12 @@ func (o *Operator) cmpNetwork(destIP interface{}) bool {
290310
}
291311

292312
func (o *Operator) matchListsCmp(msg, what string) bool {
293-
o.RLock()
294-
item, found := o.lists[what]
295-
o.RUnlock()
313+
snapshot := o.listSnapshot.Load()
314+
if snapshot == nil {
315+
return false
316+
}
317+
318+
item, found := snapshot.lists[what]
296319

297320
if found {
298321
log.Debug("%s: %s, %s", log.Red(msg), what, item)
@@ -309,7 +332,29 @@ func (o *Operator) domainsListsCmp(data string) bool {
309332
data = strings.ToLower(data)
310333
}
311334

312-
return o.matchListsCmp("domains list match", data)
335+
snapshot := o.listSnapshot.Load()
336+
if snapshot == nil {
337+
return false
338+
}
339+
340+
_, exactFound := snapshot.lists[data]
341+
342+
if exactFound {
343+
log.Debug("%s: %s", log.Red("domains list match"), data)
344+
return true
345+
}
346+
if snapshot.domainWildcards.matchesHost(data) {
347+
log.Debug("%s: %s", log.Red("domains wildcard match"), data)
348+
return true
349+
}
350+
for _, g := range snapshot.domainGlobs {
351+
if matchDomainGlob(g, data) {
352+
log.Debug("%s: %s", log.Red("domains glob match"), data)
353+
return true
354+
}
355+
}
356+
357+
return false
313358
}
314359

315360
func (o *Operator) simpleListsCmp(what string) bool {
@@ -320,17 +365,52 @@ func (o *Operator) simpleListsCmp(what string) bool {
320365
return o.matchListsCmp("simple list match", what)
321366
}
322367

323-
func (o *Operator) ipNetCmp(dstIP interface{}) bool {
324-
o.RLock()
325-
defer o.RUnlock()
368+
func (o *Operator) netListsCmp(dstIP interface{}) bool {
369+
ip := dstIP.(net.IP)
370+
ipText := ip.String()
371+
snapshot := o.listSnapshot.Load()
372+
if snapshot == nil {
373+
return false
374+
}
375+
376+
_, exactFound := snapshot.listExact[ipText]
377+
378+
if exactFound {
379+
log.Debug("%s: %s", log.Red("Net exact list match"), ipText)
380+
return true
381+
}
382+
383+
for _, netMask := range snapshot.listNets {
384+
if netMask.Contains(ip) {
385+
log.Debug("%s: %s, %s", log.Red("Net list match"), ipText, netMask.String())
386+
return true
387+
}
388+
}
389+
return false
390+
}
391+
392+
func (o *Operator) ipListsCmp(dstIP interface{}) bool {
393+
ip := dstIP.(net.IP)
394+
ipText := ip.String()
395+
snapshot := o.listSnapshot.Load()
396+
if snapshot == nil {
397+
return false
398+
}
399+
400+
_, exactFound := snapshot.listExact[ipText]
401+
402+
if exactFound {
403+
log.Debug("%s: %s", log.Red("IP list exact match"), ipText)
404+
return true
405+
}
326406

327-
for host, netMask := range o.lists {
328-
n := netMask.(*net.IPNet)
329-
if n.Contains(dstIP.(net.IP)) {
330-
log.Debug("%s: %s, %s", log.Red("Net list match"), dstIP, host)
407+
for _, netMask := range snapshot.listNets {
408+
if netMask.Contains(ip) {
409+
log.Debug("%s: %s, %s", log.Red("IP list cidr match"), ipText, netMask.String())
331410
return true
332411
}
333412
}
413+
334414
return false
335415
}
336416

@@ -341,13 +421,14 @@ func (o *Operator) reListCmp(data string) bool {
341421
if o.Sensitive == false {
342422
data = strings.ToLower(data)
343423
}
344-
o.RLock()
345-
defer o.RUnlock()
424+
snapshot := o.listSnapshot.Load()
425+
if snapshot == nil {
426+
return false
427+
}
346428

347-
for file, re := range o.lists {
348-
r := re.(*regexp.Regexp)
349-
if r.MatchString(data) {
350-
log.Debug("%s: %s, %s", log.Red("Regexp list match"), data, file)
429+
for _, entry := range snapshot.regexEntries {
430+
if entry.re.MatchString(data) {
431+
log.Debug("%s: %s, %s", log.Red("Regexp list match"), data, entry.file)
351432
return true
352433
}
353434
}
@@ -389,7 +470,7 @@ func (o *Operator) Match(con *conman.Connection, hasChecksums bool) bool {
389470
} else if o.Operand == OpDomainsLists {
390471
return o.cb(con.DstHost)
391472
} else if o.Operand == OpIPLists {
392-
return o.cb(con.DstIP.String())
473+
return o.cbGeneric(con.DstIP)
393474
} else if o.Operand == OpHashMD5Lists {
394475
return o.cb(con.Process.Checksums[procmon.HashMD5])
395476
} else if o.Operand == OpUserID || o.Operand == OpUserName {

0 commit comments

Comments
 (0)