Skip to content

Commit 5cadd0f

Browse files
hjiaweiclaude
hjiawei
and
claude
committed
logcollector: keep in_tail reading past over-long lines
At fluent-bit's in_tail defaults (buffer_max_size 32K, skip_long_lines off), the first line longer than the buffer makes in_tail abandon the whole file until it rotates. Kube audit events routinely exceed 32K (77K observed on an idle dev cluster), so kube audit collection silently stopped; fluentd had no line-length cap. Give every tail input buffer_chunk_size 256K and buffer_max_size 10M (the apiserver audit backend does not truncate; etcd caps objects at ~1.5MiB), and skip_long_lines so anything larger costs one line, never the file. Fixes EV-6835. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 987c8ca commit 5cadd0f

17 files changed

Lines changed: 531 additions & 0 deletions

‎pkg/render/logcollector/fluentbit_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,12 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() {
174174
// keeps the budget the single shared output used to have.
175175
Expect(fluentBitConf).To(ContainSubstring(`"storage.total_limit_size": "500M"`))
176176
Expect(fluentBitConf).To(ContainSubstring(`"storage.total_limit_size": "100M"`))
177+
// Long-line safety on the tail inputs: at the 32K default
178+
// buffer_max_size, one over-long line (routine for kube-audit) makes
179+
// in_tail abandon the whole file.
180+
Expect(fluentBitConf).To(ContainSubstring(`"buffer_chunk_size": "256K"`))
181+
Expect(fluentBitConf).To(ContainSubstring(`"buffer_max_size": "10M"`))
182+
Expect(fluentBitConf).To(ContainSubstring(`"skip_long_lines": true`))
177183
// No Go proxy plugins are loaded.
178184
Expect(fluentBitConf).NotTo(ContainSubstring("plugins_file"))
179185
Expect(fluentBitConf).NotTo(ContainSubstring(`"name": "linseed"`))

‎pkg/render/logcollector/pipeline.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,18 @@ func (c *fluentBitComponent) addInputs(cfg *fluentBitConfig) {
9494
"parser": in.parser,
9595
"read_from_head": true,
9696
"storage.type": "filesystem",
97+
// A line longer than buffer_max_size (32K default) makes in_tail
98+
// abandon the whole file until it rotates, and kube-audit events
99+
// routinely exceed 32K (fluentd had no line-length cap). 10M
100+
// covers anything the apiserver can log — etcd caps objects at
101+
// ~1.5MiB — and skip_long_lines confines anything larger to
102+
// dropping that line instead of the file. buffer_chunk_size is
103+
// the initial per-file buffer and the increment it grows by, so
104+
// 256K keeps a long line to a few reallocations at trivial cost
105+
// (only ~a dozen files are tailed per node).
106+
"buffer_chunk_size": "256K",
107+
"buffer_max_size": "10M",
108+
"skip_long_lines": true,
97109
})
98110
}
99111

‎pkg/render/logcollector/testdata/rendered-configs/linseed-nch.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,110 +34,146 @@
3434
"pipeline": {
3535
"inputs": [
3636
{
37+
"buffer_chunk_size": "256K",
38+
"buffer_max_size": "10M",
3739
"db": "/var/log/calico/calico-fluent-bit/in_tail_flows.db",
3840
"name": "tail",
3941
"parser": "json",
4042
"path": "/var/log/calico/flowlogs/flows.log",
4143
"read_from_head": true,
44+
"skip_long_lines": true,
4245
"storage.type": "filesystem",
4346
"tag": "flows"
4447
},
4548
{
49+
"buffer_chunk_size": "256K",
50+
"buffer_max_size": "10M",
4651
"db": "/var/log/calico/calico-fluent-bit/in_tail_dns.db",
4752
"name": "tail",
4853
"parser": "json",
4954
"path": "/var/log/calico/dnslogs/dns.log",
5055
"read_from_head": true,
56+
"skip_long_lines": true,
5157
"storage.type": "filesystem",
5258
"tag": "dns"
5359
},
5460
{
61+
"buffer_chunk_size": "256K",
62+
"buffer_max_size": "10M",
5563
"db": "/var/log/calico/calico-fluent-bit/in_tail_l7.db",
5664
"name": "tail",
5765
"parser": "json",
5866
"path": "/var/log/calico/l7logs/l7.log",
5967
"read_from_head": true,
68+
"skip_long_lines": true,
6069
"storage.type": "filesystem",
6170
"tag": "l7"
6271
},
6372
{
73+
"buffer_chunk_size": "256K",
74+
"buffer_max_size": "10M",
6475
"db": "/var/log/calico/calico-fluent-bit/in_tail_waf.db",
6576
"name": "tail",
6677
"parser": "json",
6778
"path": "/var/log/calico/waf/waf.log",
6879
"read_from_head": true,
80+
"skip_long_lines": true,
6981
"storage.type": "filesystem",
7082
"tag": "waf"
7183
},
7284
{
85+
"buffer_chunk_size": "256K",
86+
"buffer_max_size": "10M",
7387
"db": "/var/log/calico/calico-fluent-bit/in_tail_runtime.db",
7488
"name": "tail",
7589
"parser": "json",
7690
"path": "/var/log/calico/runtime-security/report.log",
7791
"read_from_head": true,
92+
"skip_long_lines": true,
7893
"storage.type": "filesystem",
7994
"tag": "runtime"
8095
},
8196
{
97+
"buffer_chunk_size": "256K",
98+
"buffer_max_size": "10M",
8299
"db": "/var/log/calico/calico-fluent-bit/in_tail_audit.tsee.db",
83100
"name": "tail",
84101
"parser": "json_audit",
85102
"path": "/var/log/calico/audit/tsee-audit.log",
86103
"read_from_head": true,
104+
"skip_long_lines": true,
87105
"storage.type": "filesystem",
88106
"tag": "audit.tsee"
89107
},
90108
{
109+
"buffer_chunk_size": "256K",
110+
"buffer_max_size": "10M",
91111
"db": "/var/log/calico/calico-fluent-bit/in_tail_audit.kube.db",
92112
"name": "tail",
93113
"parser": "json_audit",
94114
"path": "/var/log/calico/audit/kube-audit.log",
95115
"read_from_head": true,
116+
"skip_long_lines": true,
96117
"storage.type": "filesystem",
97118
"tag": "audit.kube"
98119
},
99120
{
121+
"buffer_chunk_size": "256K",
122+
"buffer_max_size": "10M",
100123
"db": "/var/log/calico/calico-fluent-bit/in_tail_bird.db",
101124
"name": "tail",
102125
"parser": "bird_regex",
103126
"path": "/var/log/calico/bird/current",
104127
"read_from_head": true,
128+
"skip_long_lines": true,
105129
"storage.type": "filesystem",
106130
"tag": "bird"
107131
},
108132
{
133+
"buffer_chunk_size": "256K",
134+
"buffer_max_size": "10M",
109135
"db": "/var/log/calico/calico-fluent-bit/in_tail_bird6.db",
110136
"name": "tail",
111137
"parser": "bird_regex",
112138
"path": "/var/log/calico/bird6/current",
113139
"read_from_head": true,
140+
"skip_long_lines": true,
114141
"storage.type": "filesystem",
115142
"tag": "bird6"
116143
},
117144
{
145+
"buffer_chunk_size": "256K",
146+
"buffer_max_size": "10M",
118147
"db": "/var/log/calico/calico-fluent-bit/in_tail_ids.events.db",
119148
"name": "tail",
120149
"parser": "json_ids_events",
121150
"path": "/var/log/calico/ids/events.log",
122151
"read_from_head": true,
152+
"skip_long_lines": true,
123153
"storage.type": "filesystem",
124154
"tag": "ids.events"
125155
},
126156
{
157+
"buffer_chunk_size": "256K",
158+
"buffer_max_size": "10M",
127159
"db": "/var/log/calico/calico-fluent-bit/in_tail_compliance.reports.db",
128160
"name": "tail",
129161
"parser": "json",
130162
"path": "/var/log/calico/compliance/compliance.*.reports.log",
131163
"read_from_head": true,
164+
"skip_long_lines": true,
132165
"storage.type": "filesystem",
133166
"tag": "compliance.reports"
134167
},
135168
{
169+
"buffer_chunk_size": "256K",
170+
"buffer_max_size": "10M",
136171
"db": "/var/log/calico/calico-fluent-bit/in_tail_policy_activity.db",
137172
"name": "tail",
138173
"parser": "json",
139174
"path": "/var/log/calico/policy/policy_activity.log",
140175
"read_from_head": true,
176+
"skip_long_lines": true,
141177
"storage.type": "filesystem",
142178
"tag": "policy_activity"
143179
},

‎pkg/render/logcollector/testdata/rendered-configs/linseed.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,110 +34,146 @@
3434
"pipeline": {
3535
"inputs": [
3636
{
37+
"buffer_chunk_size": "256K",
38+
"buffer_max_size": "10M",
3739
"db": "/var/log/calico/calico-fluent-bit/in_tail_flows.db",
3840
"name": "tail",
3941
"parser": "json",
4042
"path": "/var/log/calico/flowlogs/flows.log",
4143
"read_from_head": true,
44+
"skip_long_lines": true,
4245
"storage.type": "filesystem",
4346
"tag": "flows"
4447
},
4548
{
49+
"buffer_chunk_size": "256K",
50+
"buffer_max_size": "10M",
4651
"db": "/var/log/calico/calico-fluent-bit/in_tail_dns.db",
4752
"name": "tail",
4853
"parser": "json",
4954
"path": "/var/log/calico/dnslogs/dns.log",
5055
"read_from_head": true,
56+
"skip_long_lines": true,
5157
"storage.type": "filesystem",
5258
"tag": "dns"
5359
},
5460
{
61+
"buffer_chunk_size": "256K",
62+
"buffer_max_size": "10M",
5563
"db": "/var/log/calico/calico-fluent-bit/in_tail_l7.db",
5664
"name": "tail",
5765
"parser": "json",
5866
"path": "/var/log/calico/l7logs/l7.log",
5967
"read_from_head": true,
68+
"skip_long_lines": true,
6069
"storage.type": "filesystem",
6170
"tag": "l7"
6271
},
6372
{
73+
"buffer_chunk_size": "256K",
74+
"buffer_max_size": "10M",
6475
"db": "/var/log/calico/calico-fluent-bit/in_tail_waf.db",
6576
"name": "tail",
6677
"parser": "json",
6778
"path": "/var/log/calico/waf/waf.log",
6879
"read_from_head": true,
80+
"skip_long_lines": true,
6981
"storage.type": "filesystem",
7082
"tag": "waf"
7183
},
7284
{
85+
"buffer_chunk_size": "256K",
86+
"buffer_max_size": "10M",
7387
"db": "/var/log/calico/calico-fluent-bit/in_tail_runtime.db",
7488
"name": "tail",
7589
"parser": "json",
7690
"path": "/var/log/calico/runtime-security/report.log",
7791
"read_from_head": true,
92+
"skip_long_lines": true,
7893
"storage.type": "filesystem",
7994
"tag": "runtime"
8095
},
8196
{
97+
"buffer_chunk_size": "256K",
98+
"buffer_max_size": "10M",
8299
"db": "/var/log/calico/calico-fluent-bit/in_tail_audit.tsee.db",
83100
"name": "tail",
84101
"parser": "json_audit",
85102
"path": "/var/log/calico/audit/tsee-audit.log",
86103
"read_from_head": true,
104+
"skip_long_lines": true,
87105
"storage.type": "filesystem",
88106
"tag": "audit.tsee"
89107
},
90108
{
109+
"buffer_chunk_size": "256K",
110+
"buffer_max_size": "10M",
91111
"db": "/var/log/calico/calico-fluent-bit/in_tail_audit.kube.db",
92112
"name": "tail",
93113
"parser": "json_audit",
94114
"path": "/var/log/calico/audit/kube-audit.log",
95115
"read_from_head": true,
116+
"skip_long_lines": true,
96117
"storage.type": "filesystem",
97118
"tag": "audit.kube"
98119
},
99120
{
121+
"buffer_chunk_size": "256K",
122+
"buffer_max_size": "10M",
100123
"db": "/var/log/calico/calico-fluent-bit/in_tail_bird.db",
101124
"name": "tail",
102125
"parser": "bird_regex",
103126
"path": "/var/log/calico/bird/current",
104127
"read_from_head": true,
128+
"skip_long_lines": true,
105129
"storage.type": "filesystem",
106130
"tag": "bird"
107131
},
108132
{
133+
"buffer_chunk_size": "256K",
134+
"buffer_max_size": "10M",
109135
"db": "/var/log/calico/calico-fluent-bit/in_tail_bird6.db",
110136
"name": "tail",
111137
"parser": "bird_regex",
112138
"path": "/var/log/calico/bird6/current",
113139
"read_from_head": true,
140+
"skip_long_lines": true,
114141
"storage.type": "filesystem",
115142
"tag": "bird6"
116143
},
117144
{
145+
"buffer_chunk_size": "256K",
146+
"buffer_max_size": "10M",
118147
"db": "/var/log/calico/calico-fluent-bit/in_tail_ids.events.db",
119148
"name": "tail",
120149
"parser": "json_ids_events",
121150
"path": "/var/log/calico/ids/events.log",
122151
"read_from_head": true,
152+
"skip_long_lines": true,
123153
"storage.type": "filesystem",
124154
"tag": "ids.events"
125155
},
126156
{
157+
"buffer_chunk_size": "256K",
158+
"buffer_max_size": "10M",
127159
"db": "/var/log/calico/calico-fluent-bit/in_tail_compliance.reports.db",
128160
"name": "tail",
129161
"parser": "json",
130162
"path": "/var/log/calico/compliance/compliance.*.reports.log",
131163
"read_from_head": true,
164+
"skip_long_lines": true,
132165
"storage.type": "filesystem",
133166
"tag": "compliance.reports"
134167
},
135168
{
169+
"buffer_chunk_size": "256K",
170+
"buffer_max_size": "10M",
136171
"db": "/var/log/calico/calico-fluent-bit/in_tail_policy_activity.db",
137172
"name": "tail",
138173
"parser": "json",
139174
"path": "/var/log/calico/policy/policy_activity.log",
140175
"read_from_head": true,
176+
"skip_long_lines": true,
141177
"storage.type": "filesystem",
142178
"tag": "policy_activity"
143179
}

0 commit comments

Comments
 (0)