feat(ql): Allow sequences with multi links

Multi link sequences can define two
or more field whose values are extracted
and used as a sequence join link.

Author: rabbitstack

pkg/filter/ql/literal.go

@@ -19,14 +19,15 @@
 package ql
 
 import (
-	"github.com/rabbitstack/fibratus/pkg/event"
-	"github.com/rabbitstack/fibratus/pkg/filter/fields"
 	"net"
 	"reflect"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/rabbitstack/fibratus/pkg/event"
+	"github.com/rabbitstack/fibratus/pkg/filter/fields"
+
 	"github.com/rabbitstack/fibratus/pkg/filter/ql/functions"
 )
 
@@ -271,11 +272,11 @@ func (f *Function) validate() error {
 // SequenceExpr represents a single binary expression within the sequence.
 type SequenceExpr struct {
 	Expr Expr
-	// By contains the field literal if the sequence expression is constrained.
-	By *FieldLiteral
+	// By contains the expression link if the sequence is constrained.
+	By *SequenceLink
 	// BoundFields is a group of bound fields referenced in the sequence expression.
 	BoundFields []*BoundFieldLiteral
-	// Alias represents the sequence expression alias.
+	// Alias represents the sequence expression alias when bound fields are used.
 	Alias string
 
 	bitsets event.BitSets
@@ -381,10 +382,31 @@ func (e *SequenceExpr) HasBoundFields() bool {
 	return len(e.BoundFields) > 0
 }
 
+// SequenceLink represents a single or
+// a collection of fields that are used to
+// build the sequence join link.
+type SequenceLink struct {
+	Fields []*FieldLiteral
+}
+
+// IsCompound indicates if the sequence expression
+// uses multiple fields for the join link.
+func (l *SequenceLink) IsCompound() bool {
+	return len(l.Fields) > 1
+}
+
+// First returns the first field if the link is not compound.
+func (l *SequenceLink) First() string {
+	if len(l.Fields) == 1 {
+		return l.Fields[0].Value
+	}
+	return ""
+}
+
 // Sequence is a collection of two or more sequence expressions.
 type Sequence struct {
 	MaxSpan     time.Duration
-	By          *FieldLiteral
+	By          *SequenceLink
 	Expressions []SequenceExpr
 	IsUnordered bool
 }

pkg/filter/ql/parser.go

@@ -23,13 +23,14 @@ package ql
 import (
 	"errors"
 	"fmt"
-	"github.com/rabbitstack/fibratus/pkg/config"
-	"github.com/rabbitstack/fibratus/pkg/filter/fields"
-	"github.com/rabbitstack/fibratus/pkg/util/multierror"
 	"net"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/rabbitstack/fibratus/pkg/config"
+	"github.com/rabbitstack/fibratus/pkg/filter/fields"
+	"github.com/rabbitstack/fibratus/pkg/util/multierror"
 )
 
 // Parser builds the binary expression tree from the filter string.
@@ -71,18 +72,41 @@ func (p *Parser) ParseSequence() (*Sequence, error) {
 		p.unscan()
 	}
 
-	// parse optional global join
+	// parse optional global link
 	tok, _, _ = p.scanIgnoreWhitespace()
 	if tok == By {
 		tok, pos, lit := p.scanIgnoreWhitespace()
 		if !fields.IsField(lit) {
 			return nil, newParseError(tokstr(tok, lit), []string{"field"}, pos, p.expr)
 		}
 		var err error
-		seq.By, err = p.parseField(lit)
+		field, err := p.parseField(lit)
 		if err != nil {
 			return nil, err
 		}
+
+		seqLink := &SequenceLink{Fields: []*FieldLiteral{field}}
+
+		// handle multiple join fields separated by comma
+		for {
+			if tok, _, _ := p.scanIgnoreWhitespace(); tok != Comma {
+				p.unscan()
+				break
+			}
+
+			tok, pos, lit := p.scanIgnoreWhitespace()
+			if !fields.IsField(lit) {
+				return nil, newParseError(tokstr(tok, lit), []string{"field"}, pos, p.expr)
+			}
+			field, err := p.parseField(lit)
+			if err != nil {
+				return nil, err
+			}
+
+			seqLink.Fields = append(seqLink.Fields, field)
+		}
+
+		seq.By = seqLink
 	} else {
 		p.unscan()
 	}
@@ -127,7 +151,7 @@ func (p *Parser) ParseSequence() (*Sequence, error) {
 
 		var seqexpr SequenceExpr
 
-		// parse sequence BY or AS constraints
+		// parse sequence BY or AS constraints (links)
 		tok, _, _ = p.scanIgnoreWhitespace()
 		switch tok {
 		case By:
@@ -139,7 +163,28 @@ func (p *Parser) ParseSequence() (*Sequence, error) {
 			if err != nil {
 				return nil, err
 			}
-			seqexpr = SequenceExpr{Expr: expr, By: field}
+
+			seqLink := &SequenceLink{Fields: []*FieldLiteral{field}}
+
+			// handle multiple join fields separated by comma
+			for {
+				if tok, _, _ := p.scanIgnoreWhitespace(); tok != Comma {
+					p.unscan()
+					break
+				}
+
+				tok, pos, lit := p.scanIgnoreWhitespace()
+				if !fields.IsField(lit) {
+					return nil, newParseError(tokstr(tok, lit), []string{"field"}, pos, p.expr)
+				}
+				field, err := p.parseField(lit)
+				if err != nil {
+					return nil, err
+				}
+
+				seqLink.Fields = append(seqLink.Fields, field)
+			}
+			seqexpr = SequenceExpr{Expr: expr, By: seqLink}
 		case As:
 			tok, pos, lit := p.scanIgnoreWhitespace()
 			if tok != Ident {

pkg/filter/ql/parser_test.go

@@ -20,6 +20,8 @@ package ql
 
 import (
 	"errors"
+	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -271,6 +273,31 @@ func TestParseSequence(t *testing.T) {
 			time.Duration(0),
 			true,
 		},
+		{
+			`|evt.name = 'CreateProcess'| by ps.exe, ps.uuid
+			 |evt.name = 'CreateFile'| by file.name, ps.uuid
+			`,
+			nil,
+			time.Duration(0),
+			true,
+		},
+		{
+			`by ps.exe, ps.uuid
+			 |evt.name = 'CreateProcess'|
+			 |evt.name = 'CreateFile'|
+			`,
+			nil,
+			time.Duration(0),
+			true,
+		},
+		{
+			`|evt.name = 'CreateProcess'| by ps.exe, 
+			 |evt.name = 'CreateFile'| by file.name, ps.uuid
+			`,
+			errors.New("expected field"),
+			time.Duration(0),
+			true,
+		},
 		{
 
 			`by ps.pid
@@ -336,7 +363,7 @@ func TestParseSequence(t *testing.T) {
 			 |evt.name = 'CreateProcess'| as e1
 			 |evt.name = 'CreateFile' and $e1.ps.ame = file.name |
 			`,
-			errors.New("expected field after bound ref"),
+			errors.New("expected field/segment after bound ref"),
 			time.Second * 30,
 			false,
 		},
@@ -352,8 +379,8 @@ func TestParseSequence(t *testing.T) {
 		},
 		{
 
-			`by ps.uuid
-			 maxspan 2m
+			`maxspan 2m
+			 by ps.uuid
 			 |evt.name = 'CreateProcess'| by ps.uuid
 			 |evt.name = 'CreateFile'| by ps.uuid
 			`,
@@ -372,6 +399,10 @@ func TestParseSequence(t *testing.T) {
 			t.Errorf("%d. exp=%s got error=\n%v", i, tt.expr, err)
 		}
 
+		if err != nil && tt.err != nil {
+			assert.True(t, strings.Contains(err.Error(), tt.err.Error()), fmt.Sprintf("error '%v' should contain '%v'", err, tt.err))
+		}
+
 		if seq != nil {
 			if seq.MaxSpan != tt.maxSpan {
 				t.Errorf("%d. exp=%s maxspan=%s got maxspan=%v", i, tt.expr, tt.maxSpan, seq.MaxSpan)

pkg/rules/sequence_test.go

@@ -443,6 +443,56 @@ func TestSimpleSequenceDeadline(t *testing.T) {
 	require.True(t, ss.runSequence(e2))
 }
 
+func TestSequenceMultiLinks(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+
+	c := &config.FilterConfig{Name: "Command shell created a temp file"}
+	f := filter.New(`
+	sequence
+	maxspan 100ms
+  	|evt.name = 'CreateProcess' and ps.name = 'cmd.exe'| by ps.exe, ps.pid
+  	|evt.name = 'CreateFile' and file.path icontains 'temp'| by file.path, ps.pid
+	`, &config.Config{EventSource: config.EventSourceConfig{EnableFileIOEvents: true}, Filters: &config.Filters{}})
+	require.NoError(t, f.Compile())
+
+	ss := newSequenceState(f, c, new(ps.SnapshotterMock))
+
+	e1 := &event.Event{
+		Type:      event.CreateProcess,
+		Timestamp: time.Now(),
+		Name:      "CreateProcess",
+		Tid:       2484,
+		PID:       859,
+		PS: &pstypes.PS{
+			Name: "cmd.exe",
+			Exe:  "C:\\Windows\\system32\\svchost-temp.exe",
+		},
+		Params: event.Params{
+			params.ProcessID: {Name: params.ProcessID, Type: params.Uint32, Value: uint32(4143)},
+		},
+		Metadata: map[event.MetadataKey]any{"foo": "bar", "fooz": "barzz"},
+	}
+	require.False(t, ss.runSequence(e1))
+
+	e2 := &event.Event{
+		Type:      event.CreateFile,
+		Timestamp: time.Now(),
+		Name:      "CreateFile",
+		Tid:       2484,
+		PID:       859,
+		Category:  event.File,
+		PS: &pstypes.PS{
+			Name: "cmd.exe",
+			Exe:  "C:\\Windows\\system32\\svchost.exe",
+		},
+		Params: event.Params{
+			params.FilePath: {Name: params.FilePath, Type: params.UnicodeString, Value: "C:\\Windows\\system32\\svchost-temp.exe"},
+		},
+		Metadata: map[event.MetadataKey]any{"foo": "bar", "fooz": "barzz"},
+	}
+	require.True(t, ss.runSequence(e2))
+}
+
 func TestComplexSequence(t *testing.T) {
 	log.SetLevel(log.DebugLevel)