Skip to content

Commit 9e95d69

Browse files
rabbitstackrabbitstack
authored
chore(pe): Replace standard library PE parser with saferwall parser (#161)
* use saferwall/pe package for PE introspection * avoid returning lang id, introduce just string sanity check. Expand test cases and other minor adjustments * introduce VA (Virtual Address) region area reading and `ParseMem` function for fetching remote PE * introduce ParseMem tests and small adjustments * VA region tests, upgrade saferwall/pe and set up options for directory parsing * document structures/functions * update the stats structure with new expvar metrics * address lint warnings and adjust tests * fix lint warnings in tests
1 parent ae9939d commit 9e95d69

21 files changed

Lines changed: 1249 additions & 1077 deletions

File tree

‎cmd/fibratus/app/stats.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,9 @@ type Stats struct {
9696
OutputAMQPPublishErrors int `json:"output.amqp.publish.errors"`
9797
OutputConsoleErrors int `json:"output.console.errors"`
9898
OutputNullBlackholeEvents int `json:"output.null.blackhole.events"`
99-
PeFailedResourceEntryReads int `json:"pe.failed.resource.entry.reads"`
100-
PeMaxResourceEntriesExceeded int `json:"pe.max.resource.entries.exceeded"`
99+
PeSkippedImages int `json:"pe.skipped.images"`
100+
PeDirectoryParseErrors int `json:"pe.directory.parse.errors"`
101+
PeVersionResourcesParseErrors int `json:"pe.version.resources.parse.errors"`
101102
ProcessCount int `json:"process.count"`
102103
ProcessModuleCount int `json:"process.module.count"`
103104
ProcessLookupFailureCount map[int]int `json:"process.lookup.failure.count"`

‎go.mod‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ require (
1717
github.com/pkg/errors v0.9.1
1818
github.com/qmuntal/stateless v1.6.0
1919
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
20+
github.com/saferwall/pe v1.4.0
2021
github.com/sirupsen/logrus v1.4.1
2122
github.com/spf13/cobra v0.0.3
2223
github.com/spf13/pflag v1.0.5
@@ -39,6 +40,7 @@ require (
3940
github.com/Masterminds/semver/v3 v3.1.1 // indirect
4041
github.com/antchfx/xpath v1.2.1 // indirect
4142
github.com/davecgh/go-spew v1.1.1 // indirect
43+
github.com/edsrzf/mmap-go v1.1.0 // indirect
4244
github.com/fatih/color v1.7.0 // indirect
4345
github.com/fsnotify/fsnotify v1.4.7 // indirect
4446
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
@@ -65,6 +67,7 @@ require (
6567
github.com/subosito/gotenv v1.2.0 // indirect
6668
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
6769
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
70+
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
6871
golang.org/x/crypto v0.1.0 // indirect
6972
golang.org/x/net v0.7.0 // indirect
7073
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect

‎go.sum‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
3636
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
3737
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
3838
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
39+
github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
40+
github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
3941
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
4042
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
4143
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@@ -153,6 +155,8 @@ github.com/qmuntal/stateless v1.6.0/go.mod h1:cWTwXu9ey+FxI0fHvDi1nGCtpYa8N1X2aO
153155
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
154156
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
155157
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
158+
github.com/saferwall/pe v1.4.0 h1:5aZVTAEqYYwbC078GDPtml6pVcY6iLbTgO86leqp6K8=
159+
github.com/saferwall/pe v1.4.0/go.mod h1:Fh6INnfm6vUTCW1w/pZ59aWQhlOwqBVxVACtadb3UbA=
156160
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
157161
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
158162
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -212,6 +216,8 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:
212216
github.com/yuin/goldmark v1.5.2 h1:ALmeCk/px5FSm1MAcFBAsVKZjDuMVj8Tm7FFIlMJnqU=
213217
github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
214218
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
219+
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak=
220+
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
215221
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
216222
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
217223
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -254,6 +260,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
254260
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
255261
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
256262
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
263+
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
257264
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
258265
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
259266
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

‎pkg/filter/ql/functions/entropy.go‎

Lines changed: 12 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,15 @@ package functions
2020

2121
import (
2222
"fmt"
23-
"math"
23+
"github.com/rabbitstack/fibratus/pkg/util/entropy"
24+
"strings"
2425
)
2526

2627
const (
27-
shannonAlgo = "shannon"
28+
// Shannon computes the string entropy by employing
29+
// the Shannon algorithm.
30+
// https://en.wikipedia.org/wiki/Entropy_(information_theory)
31+
Shannon = "shannon"
2832
)
2933

3034
// Entropy measures the string entropy
@@ -36,12 +40,12 @@ func (f Entropy) Call(args []interface{}) (interface{}, bool) {
3640
}
3741
s := parseString(0, args)
3842
if len(args) == 1 {
39-
return shannon(s), true
43+
return entropy.Shannon(s), true
4044
}
4145
algo := parseString(1, args)
4246
switch algo {
43-
case shannonAlgo:
44-
return shannon(s), true
47+
case Shannon:
48+
return entropy.Shannon(s), true
4549
default:
4650
return false, false
4751
}
@@ -58,8 +62,9 @@ func (f Entropy) Desc() FunctionDesc {
5862
if len(args) == 1 {
5963
return nil
6064
}
61-
if len(args) > 1 && args[1] != shannonAlgo {
62-
return fmt.Errorf("unsupported entropy algorithm: %s. Availiable algorithms: shannon", args[1])
65+
if len(args) > 1 && args[1] != Shannon {
66+
return fmt.Errorf("unsupported entropy algorithm: %s. Availiable algorithms: %s", args[1],
67+
strings.Join([]string{Shannon}, "|"))
6368
}
6469
return nil
6570
},
@@ -68,22 +73,3 @@ func (f Entropy) Desc() FunctionDesc {
6873
}
6974

7075
func (f Entropy) Name() Fn { return EntropyFn }
71-
72-
// shannon measures the Shannon entropy of a string.
73-
func shannon(value string) int {
74-
frq := make(map[rune]float64)
75-
76-
//get frequency of characters
77-
for _, i := range value {
78-
frq[i]++
79-
}
80-
81-
var sum float64
82-
83-
for _, v := range frq {
84-
f := v / float64(len(value))
85-
sum += f * math.Log2(f)
86-
}
87-
88-
return int(math.Ceil(sum*-1)) * len(value)
89-
}

‎pkg/pe/config.go‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ package pe
2424
import (
2525
"github.com/spf13/pflag"
2626
"github.com/spf13/viper"
27-
"path/filepath"
28-
"strings"
2927
)
3028

3129
const (
@@ -62,13 +60,3 @@ func AddFlags(flags *pflag.FlagSet) {
6260
flags.Bool(readSections, false, "Indicates if full section inspection is allowed")
6361
flags.StringSlice(excludedImages, []string{}, "Contains a list of comma-separated images names that are excluded from PE parsing")
6462
}
65-
66-
// ShouldSkipProcess determines whether the specified filename name is ignored by PE reader.
67-
func (c Config) shouldSkipImage(filename string) bool {
68-
for _, img := range c.ExcludedImages {
69-
if strings.EqualFold(img, filepath.Base(filename)) {
70-
return true
71-
}
72-
}
73-
return false
74-
}

‎pkg/pe/doc.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@
1717
*/
1818

1919
// Package pe contains different facilities for dealing with Portable Executable specifics and digging out valuable insights
20-
// from it.
20+
// from PE.
2121
package pe

‎pkg/pe/entropy.go‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

0 commit comments

Comments
 (0)