Skip to content

Commit 0a53612

Browse files
jorbaumjorbaum
committed
Only exclude filter shall pass unknown source type
Also rename remaining instances of log type to source type
1 parent e4d6fb6 commit 0a53612

7 files changed

Lines changed: 142 additions & 121 deletions

File tree

‎src/pkg/egress/syslog/filtering_drain_writer.go‎

Lines changed: 3 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package syslog
22

33
import (
44
"errors"
5-
"strings"
65

76
"code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2"
87
"code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress"
@@ -51,11 +50,7 @@ func (w *FilteringDrainWriter) Write(env *loggregator_v2.Envelope) error {
5150
}
5251
}
5352
if env.GetLog() != nil {
54-
sourceType, ok := env.GetTags()["source_type"]
55-
if !ok {
56-
// Default to sending logs if no source_type tag is present
57-
sourceType = ""
58-
}
53+
sourceType := env.GetTags()["source_type"]
5954
if sendsLogs(w.binding.DrainData, w.binding.LogFilter, sourceType) {
6055
return w.writer.Write(env)
6156
}
@@ -69,40 +64,12 @@ func (w *FilteringDrainWriter) Write(env *loggregator_v2.Envelope) error {
6964
return nil
7065
}
7166

72-
// shouldIncludeLog determines if a log with the given sourceTypeTag should be forwarded
73-
func shouldIncludeLog(logFilter *SourceTypeSet, sourceTypeTag string) bool {
74-
// Empty filter or missing source type means no filtering
75-
if logFilter == nil || sourceTypeTag == "" {
76-
return true
77-
}
78-
79-
// Find the first "/" to extract prefix
80-
idx := strings.IndexByte(sourceTypeTag, '/')
81-
prefix := sourceTypeTag
82-
if idx != -1 {
83-
prefix = sourceTypeTag[:idx]
84-
}
85-
86-
// Prefer map lookup over switch for performance
87-
logType := SourceType(prefix)
88-
if !logType.IsValid() {
89-
// Unknown source type, default to not filtering
90-
return true
91-
}
92-
93-
return logFilter.Contains(logType)
94-
}
95-
96-
func sendsLogs(drainData DrainData, logFilter *SourceTypeSet, sourceTypeTag string) bool {
67+
func sendsLogs(drainData DrainData, logFilter *LogFilter, sourceTypeTag string) bool {
9768
if drainData != LOGS && drainData != LOGS_AND_METRICS && drainData != LOGS_NO_EVENTS {
9869
return false
9970
}
10071

101-
if shouldIncludeLog(logFilter, sourceTypeTag) {
102-
return true
103-
}
104-
105-
return false
72+
return logFilter.ShouldInclude(sourceTypeTag)
10673
}
10774

10875
func sendsMetrics(drainData DrainData) bool {

‎src/pkg/egress/syslog/filtering_drain_writer_test.go‎

Lines changed: 44 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -54,36 +54,57 @@ var _ = Describe("Filtering Drain Writer", func() {
5454
Expect(err).To(HaveOccurred())
5555
})
5656

57-
It("sends logs when source_type tag is missing", func() {
58-
binding := syslog.Binding{
59-
DrainData: syslog.LOGS,
60-
}
61-
fakeWriter := &fakeWriter{}
62-
drainWriter, err := syslog.NewFilteringDrainWriter(binding, fakeWriter)
63-
Expect(err).NotTo(HaveOccurred())
57+
Context("when source_type tag is missing", func() {
58+
var envelope *loggregator_v2.Envelope
6459

65-
envelope := &loggregator_v2.Envelope{
66-
Message: &loggregator_v2.Envelope_Log{
67-
Log: &loggregator_v2.Log{
68-
Payload: []byte("test log"),
60+
BeforeEach(func() {
61+
envelope = &loggregator_v2.Envelope{
62+
Message: &loggregator_v2.Envelope_Log{
63+
Log: &loggregator_v2.Log{
64+
Payload: []byte("test log"),
65+
},
6966
},
70-
},
71-
Tags: map[string]string{
72-
// source_type tag is intentionally missing
73-
},
74-
}
67+
Tags: map[string]string{
68+
// source_type tag is intentionally missing
69+
},
70+
}
71+
})
7572

76-
err = drainWriter.Write(envelope)
73+
It("omits logs when source type include filter is configured with LOGS", func() {
74+
binding := syslog.Binding{
75+
DrainData: syslog.LOGS,
76+
LogFilter: syslog.NewLogFilter(syslog.SourceTypeSet{syslog.SOURCE_APP: struct{}{}}, syslog.LogFilterModeInclude),
77+
}
78+
fakeWriter := &fakeWriter{}
79+
drainWriter, err := syslog.NewFilteringDrainWriter(binding, fakeWriter)
80+
Expect(err).NotTo(HaveOccurred())
7781

78-
Expect(err).NotTo(HaveOccurred())
79-
Expect(fakeWriter.received).To(Equal(1))
82+
err = drainWriter.Write(envelope)
83+
84+
Expect(err).NotTo(HaveOccurred())
85+
Expect(fakeWriter.received).To(Equal(0))
86+
})
87+
88+
It("sends logs when source type exclude filter is configured with LOGS", func() {
89+
binding := syslog.Binding{
90+
DrainData: syslog.LOGS,
91+
LogFilter: syslog.NewLogFilter(syslog.SourceTypeSet{syslog.SOURCE_RTR: struct{}{}}, syslog.LogFilterModeExclude),
92+
}
93+
fakeWriter := &fakeWriter{}
94+
drainWriter, err := syslog.NewFilteringDrainWriter(binding, fakeWriter)
95+
Expect(err).NotTo(HaveOccurred())
96+
97+
err = drainWriter.Write(envelope)
98+
99+
Expect(err).NotTo(HaveOccurred())
100+
Expect(fakeWriter.received).To(Equal(1))
101+
})
80102
})
81103

82104
It("filters logs based on include filter - includes only APP logs", func() {
83-
appFilter := syslog.SourceTypeSet{syslog.SOURCE_APP: struct{}{}}
84105
binding := syslog.Binding{
85106
DrainData: syslog.LOGS,
86-
LogFilter: &appFilter,
107+
LogFilter: syslog.NewLogFilter(syslog.SourceTypeSet{syslog.SOURCE_APP: struct{}{}}, syslog.LogFilterModeInclude),
87108
}
88109
fakeWriter := &fakeWriter{}
89110
drainWriter, err := syslog.NewFilteringDrainWriter(binding, fakeWriter)
@@ -120,14 +141,9 @@ var _ = Describe("Filtering Drain Writer", func() {
120141
})
121142

122143
It("filters logs based on exclude filter - excludes RTR logs", func() {
123-
// Include APP and STG, effectively excluding RTR
124-
includeFilter := syslog.SourceTypeSet{
125-
syslog.SOURCE_APP: struct{}{},
126-
syslog.SOURCE_STG: struct{}{},
127-
}
128144
binding := syslog.Binding{
129145
DrainData: syslog.LOGS,
130-
LogFilter: &includeFilter,
146+
LogFilter: syslog.NewLogFilter(syslog.SourceTypeSet{syslog.SOURCE_RTR: struct{}{}}, syslog.LogFilterModeExclude),
131147
}
132148
fakeWriter := &fakeWriter{}
133149
drainWriter, err := syslog.NewFilteringDrainWriter(binding, fakeWriter)
@@ -164,10 +180,9 @@ var _ = Describe("Filtering Drain Writer", func() {
164180
})
165181

166182
It("sends logs with unknown source_type prefix when filter is set", func() {
167-
appFilter := syslog.SourceTypeSet{syslog.SOURCE_APP: struct{}{}}
168183
binding := syslog.Binding{
169184
DrainData: syslog.LOGS,
170-
LogFilter: &appFilter,
185+
LogFilter: syslog.NewLogFilter(syslog.SourceTypeSet{syslog.SOURCE_APP: struct{}{}}, syslog.LogFilterModeInclude),
171186
}
172187
fakeWriter := &fakeWriter{}
173188
drainWriter, err := syslog.NewFilteringDrainWriter(binding, fakeWriter)

‎src/pkg/egress/syslog/source.go‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ func (lt SourceType) IsValid() bool {
3434
return ok
3535
}
3636

37+
// ParseSourceType parses a string into a SourceType value
38+
func ParseSourceType(s string) (SourceType, bool) {
39+
lt := SourceType(strings.ToUpper(s))
40+
return lt, lt.IsValid()
41+
}
42+
3743
// AllSourceTypes returns all valid source types
3844
func AllSourceTypes() []SourceType {
3945
types := make([]SourceType, 0, len(validSourceTypes))
@@ -43,6 +49,14 @@ func AllSourceTypes() []SourceType {
4349
return types
4450
}
4551

52+
// ExtractPrefix extracts the prefix from a source_type tag (e.g., "APP/PROC/WEB/0" -> "APP")
53+
func ExtractPrefix(sourceTypeTag string) string {
54+
if idx := strings.IndexByte(sourceTypeTag, '/'); idx != -1 {
55+
return sourceTypeTag[:idx]
56+
}
57+
return sourceTypeTag
58+
}
59+
4660
// SourceTypeSet is a set of SourceTypes for efficient membership checking
4761
type SourceTypeSet map[SourceType]struct{}
4862

@@ -57,8 +71,50 @@ func (s SourceTypeSet) Contains(lt SourceType) bool {
5771
return exists
5872
}
5973

60-
// ParseSourceType parses a string into a SourceType value
61-
func ParseSourceType(s string) (SourceType, bool) {
62-
lt := SourceType(strings.ToUpper(s))
63-
return lt, lt.IsValid()
74+
// LogFilterMode determines how the log filter should be applied
75+
type LogFilterMode int
76+
77+
const (
78+
// LogFilterModeInclude only includes logs matching the specified types (strict)
79+
LogFilterModeInclude LogFilterMode = iota
80+
// LogFilterModeExclude excludes logs matching the specified types (permissive)
81+
LogFilterModeExclude
82+
)
83+
84+
// LogFilter encapsulates source type filtering configuration
85+
type LogFilter struct {
86+
Types SourceTypeSet
87+
Mode LogFilterMode
88+
}
89+
90+
// NewLogFilter creates a new LogFilter with the given types and mode
91+
func NewLogFilter(types SourceTypeSet, mode LogFilterMode) *LogFilter {
92+
return &LogFilter{
93+
Types: types,
94+
Mode: mode,
95+
}
96+
}
97+
98+
// ShouldInclude determines if a log with the given sourceTypeTag should be forwarded
99+
// Include mode omits missing/unknown source types, exclude mode forwards them
100+
func (f *LogFilter) ShouldInclude(sourceTypeTag string) bool {
101+
if f == nil {
102+
return true
103+
}
104+
105+
if sourceTypeTag == "" {
106+
return f.Mode == LogFilterModeExclude
107+
}
108+
109+
prefix := ExtractPrefix(sourceTypeTag)
110+
sourceType := SourceType(prefix)
111+
if !sourceType.IsValid() {
112+
return f.Mode == LogFilterModeExclude
113+
}
114+
115+
inSet := f.Types.Contains(sourceType)
116+
if f.Mode == LogFilterModeInclude {
117+
return inSet
118+
}
119+
return !inSet
64120
}

‎src/pkg/egress/syslog/syslog_connector.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ type Binding struct {
2020
DrainData DrainData `json:"type,omitempty"`
2121
OmitMetadata bool
2222
InternalTls bool
23-
LogFilter *SourceTypeSet
23+
LogFilter *LogFilter
2424
}
2525

2626
type Drain struct {

‎src/pkg/ingress/bindings/binding_config.go‎

Lines changed: 13 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -86,49 +86,35 @@ func getBindingType(u *url.URL) syslog.DrainData {
8686
return drainData
8787
}
8888

89-
func (d *DrainParamParser) getLogFilter(u *url.URL) *syslog.SourceTypeSet {
89+
func (d *DrainParamParser) getLogFilter(u *url.URL) *syslog.LogFilter {
9090
includeSourceTypes := u.Query().Get("include-source-types")
9191
excludeSourceTypes := u.Query().Get("exclude-source-types")
9292

9393
if excludeSourceTypes != "" {
94-
return d.NewSourceTypeSet(excludeSourceTypes, true)
94+
return d.newLogFilter(excludeSourceTypes, syslog.LogFilterModeExclude)
9595
} else if includeSourceTypes != "" {
96-
return d.NewSourceTypeSet(includeSourceTypes, false)
96+
return d.newLogFilter(includeSourceTypes, syslog.LogFilterModeInclude)
9797
}
9898
return nil
9999
}
100100

101-
// NewSourceTypeSet parses a URL query parameter into a Set of SourceTypes.
102-
// logTypeList is assumed to be a comma-separated list of valid source types.
103-
func (d *DrainParamParser) NewSourceTypeSet(logTypeList string, isExclude bool) *syslog.SourceTypeSet {
104-
if logTypeList == "" {
101+
// newLogFilter parses a URL query parameter into a LogFilter.
102+
// sourceTypeList is assumed to be a comma-separated list of valid source types.
103+
func (d *DrainParamParser) newLogFilter(sourceTypeList string, mode syslog.LogFilterMode) *syslog.LogFilter {
104+
if sourceTypeList == "" {
105105
return nil
106106
}
107107

108-
logTypes := strings.Split(logTypeList, ",")
109-
set := make(syslog.SourceTypeSet, len(logTypes))
108+
sourceTypes := strings.Split(sourceTypeList, ",")
109+
set := make(syslog.SourceTypeSet, len(sourceTypes))
110110

111-
for _, logType := range logTypes {
112-
logType = strings.TrimSpace(logType)
113-
t, _ := syslog.ParseSourceType(logType)
111+
for _, sourceType := range sourceTypes {
112+
sourceType = strings.TrimSpace(sourceType)
113+
t, _ := syslog.ParseSourceType(sourceType)
114114
set.Add(t)
115115
}
116116

117-
if isExclude {
118-
// Invert the set - include all types except those in the set
119-
fullSet := make(syslog.SourceTypeSet)
120-
121-
for _, t := range syslog.AllSourceTypes() {
122-
fullSet.Add(t)
123-
}
124-
125-
for t := range set {
126-
delete(fullSet, t)
127-
}
128-
return &fullSet
129-
}
130-
131-
return &set
117+
return syslog.NewLogFilter(set, mode)
132118
}
133119

134120
func getRemoveMetadataQuery(u *url.URL) string {

‎src/pkg/ingress/bindings/binding_config_test.go‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -119,32 +119,32 @@ var _ = Describe("Drain Param Config", func() {
119119
testCases := []struct {
120120
name string
121121
url string
122-
expected *syslog.SourceTypeSet
122+
expected *syslog.LogFilter
123123
}{
124124
{
125125
name: "empty drain URL defaults to all types",
126126
url: "https://test.org/drain",
127-
expected: NewSourceTypeSet(),
127+
expected: nil,
128128
},
129129
{
130130
name: "include-source-types=app",
131131
url: "https://test.org/drain?include-source-types=app",
132-
expected: NewSourceTypeSet(syslog.SOURCE_APP),
132+
expected: NewLogFilter(syslog.LogFilterModeInclude, syslog.SOURCE_APP),
133133
},
134134
{
135135
name: "include-source-types=app,stg,cell",
136136
url: "https://test.org/drain?include-source-types=app,stg,cell",
137-
expected: NewSourceTypeSet(syslog.SOURCE_APP, syslog.SOURCE_STG, syslog.SOURCE_CELL),
137+
expected: NewLogFilter(syslog.LogFilterModeInclude, syslog.SOURCE_APP, syslog.SOURCE_STG, syslog.SOURCE_CELL),
138138
},
139139
{
140140
name: "exclude-source-types=rtr,cell,stg",
141141
url: "https://test.org/drain?exclude-source-types=rtr,cell,stg",
142-
expected: NewSourceTypeSet(syslog.SOURCE_API, syslog.SOURCE_LGR, syslog.SOURCE_APP, syslog.SOURCE_SSH),
142+
expected: NewLogFilter(syslog.LogFilterModeExclude, syslog.SOURCE_RTR, syslog.SOURCE_CELL, syslog.SOURCE_STG),
143143
},
144144
{
145145
name: "exclude-source-types=rtr",
146146
url: "https://test.org/drain?exclude-source-types=rtr",
147-
expected: NewSourceTypeSet(syslog.SOURCE_API, syslog.SOURCE_STG, syslog.SOURCE_LGR, syslog.SOURCE_APP, syslog.SOURCE_SSH, syslog.SOURCE_CELL),
147+
expected: NewLogFilter(syslog.LogFilterModeExclude, syslog.SOURCE_RTR),
148148
},
149149
}
150150

@@ -252,13 +252,10 @@ func (f *stubFetcher) DrainLimit() int {
252252
return -1
253253
}
254254

255-
func NewSourceTypeSet(logTypes ...syslog.SourceType) *syslog.SourceTypeSet {
256-
if len(logTypes) == 0 {
257-
return nil
258-
}
259-
set := make(syslog.SourceTypeSet, len(logTypes))
260-
for _, t := range logTypes {
255+
func NewLogFilter(mode syslog.LogFilterMode, sourceTypes ...syslog.SourceType) *syslog.LogFilter {
256+
set := make(syslog.SourceTypeSet, len(sourceTypes))
257+
for _, t := range sourceTypes {
261258
set[t] = struct{}{}
262259
}
263-
return &set
260+
return syslog.NewLogFilter(set, mode)
264261
}

0 commit comments

Comments
 (0)