Skip to content

Commit 732215e

Browse files
committed
feat(errortracking): extend agenttelemetry component with error log submission
- Add SubmitErrorLog(ErrorLog) to the component interface - Add bounded errLogsCh channel, non-blocking SubmitErrorLog(), and flushErrortracking() scheduler job - Add errortracking_sender.go: converts ErrorLog to dd-go Log with stack symbolization - Add logs_payload.go: builds LogsPayload batches for HTTP POST - Refactor sender.go: extract shared sendPayloadBody() helper reused by metrics and logs paths
1 parent d58e35d commit 732215e

14 files changed

Lines changed: 303 additions & 47 deletions

File tree

comp/core/agenttelemetry/def/BUILD.bazel

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,8 @@ go_library(
55
srcs = ["component.go"],
66
importpath = "github.com/DataDog/datadog-agent/comp/core/agenttelemetry/def",
77
visibility = ["//visibility:public"],
8-
deps = ["//pkg/fleet/installer/telemetry"],
8+
deps = [
9+
"//pkg/fleet/installer/telemetry",
10+
"//pkg/util/log/errortracking",
11+
],
912
)

comp/core/agenttelemetry/def/component.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"context"
1111

1212
installertelemetry "github.com/DataDog/datadog-agent/pkg/fleet/installer/telemetry"
13+
"github.com/DataDog/datadog-agent/pkg/util/log/errortracking"
1314
)
1415

1516
// team: agent-runtimes
@@ -21,5 +22,26 @@ type Component interface {
2122
// payload - de-serializable into JSON
2223
SendEvent(eventType string, eventPayload []byte) error
2324

25+
// SubmitErrorLog accepts a single error log record from the
26+
// pkg/util/log slog handler and enqueues it for asynchronous flush
27+
// to the internal agent telemetry intake. Implementations
28+
// MUST be non-blocking on the hot path: enqueue to a bounded buffer
29+
// and drop silently on overflow.
30+
//
31+
// Recursion prevention: the flush path
32+
// (sendLogsBatch → sendPayload) MUST NOT log at
33+
// Error or above. A flush-path Errorf would re-enter the slog
34+
// handler and feed records back into this same channel. This
35+
// invariant is enforced by convention — there is no runtime
36+
// caller-identity guard. See
37+
// comp/core/agenttelemetry/impl/errortracking_sender.go. If a
38+
// flush fails, the failed batch is not re-attempted; the Debug-level
39+
// log is the only signal.
40+
//
41+
// This method receives the ErrorLog value-type defined at
42+
// pkg/util/log/errortracking; the component never sees raw slog
43+
// types on its public surface.
44+
SubmitErrorLog(log errortracking.ErrorLog)
45+
2446
StartStartupSpan(operationName string) (*installertelemetry.Span, context.Context)
2547
}

comp/core/agenttelemetry/def/go.mod

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ module github.com/DataDog/datadog-agent/comp/core/agenttelemetry/def
22

33
go 1.25.0
44

5-
require github.com/DataDog/datadog-agent/pkg/fleet/installer v0.70.0
5+
require (
6+
github.com/DataDog/datadog-agent/pkg/fleet/installer v0.70.0
7+
github.com/DataDog/datadog-agent/pkg/util/log v0.73.2
8+
)
69

710
require (
811
github.com/DataDog/datadog-agent/pkg/template v0.73.2 // indirect
9-
github.com/DataDog/datadog-agent/pkg/util/log v0.73.2 // indirect
1012
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.73.2 // indirect
1113
github.com/DataDog/datadog-agent/pkg/version v0.73.2 // indirect
1214
github.com/ebitengine/purego v0.10.0 // indirect

comp/core/agenttelemetry/fx/go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ require (
4747
github.com/DataDog/datadog-agent/pkg/util/hostinfo v0.0.0-20251027120702-0e91eee9852f // indirect
4848
github.com/DataDog/datadog-agent/pkg/util/http v0.75.4 // indirect
4949
github.com/DataDog/datadog-agent/pkg/util/log v0.75.4 // indirect
50+
github.com/DataDog/datadog-agent/pkg/util/log/setup v0.0.0-00010101000000-000000000000 // indirect
5051
github.com/DataDog/datadog-agent/pkg/util/option v0.75.4 // indirect
5152
github.com/DataDog/datadog-agent/pkg/util/pointer v0.75.4 // indirect
5253
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.75.4 // indirect

comp/core/agenttelemetry/impl/BUILD.bazel

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ go_library(
55
srcs = [
66
"agenttelemetry.go",
77
"config.go",
8+
"errortracking_sender.go",
9+
"logs_payload.go",
810
"runner.go",
911
"sender.go",
1012
"utils.go",
@@ -24,32 +26,41 @@ go_library(
2426
"//pkg/fleet/installer/telemetry",
2527
"//pkg/util/hostinfo",
2628
"//pkg/util/http",
29+
"//pkg/util/log/errortracking",
30+
"//pkg/util/log/setup",
2731
"//pkg/util/scrubber",
2832
"//pkg/version",
2933
"@com_github_datadog_zstd//:zstd",
3034
"@com_github_prometheus_client_model//go",
3135
"@com_github_robfig_cron_v3//:cron",
3236
"@in_yaml_go_yaml_v2//:yaml",
37+
"@org_uber_go_atomic//:atomic",
3338
],
3439
)
3540

3641
go_test(
3742
name = "impl_test",
38-
srcs = ["agenttelemetry_test.go"],
43+
srcs = [
44+
"agenttelemetry_test.go",
45+
"errortracking_sender_test.go",
46+
],
3947
embed = [":impl"],
4048
gotags = ["test"],
4149
deps = [
4250
"//comp/core/log/def",
4351
"//comp/core/log/mock",
4452
"//comp/core/telemetry/def",
4553
"//comp/core/telemetry/mock",
54+
"//comp/logs/agent/config",
4655
"//pkg/config/mock",
4756
"//pkg/util/fxutil",
4857
"//pkg/util/jsonquery",
58+
"//pkg/util/log/errortracking",
4959
"@com_github_datadog_zstd//:zstd",
5060
"@com_github_prometheus_client_model//go",
5161
"@com_github_stretchr_testify//assert",
5262
"@com_github_stretchr_testify//mock",
5363
"@com_github_stretchr_testify//require",
64+
"@org_uber_go_atomic//:atomic",
5465
],
5566
)

comp/core/agenttelemetry/impl/agenttelemetry.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/DataDog/datadog-agent/pkg/config/utils"
3030
installertelemetry "github.com/DataDog/datadog-agent/pkg/fleet/installer/telemetry"
3131
httputils "github.com/DataDog/datadog-agent/pkg/util/http"
32+
"github.com/DataDog/datadog-agent/pkg/util/log/errortracking"
3233
"github.com/DataDog/datadog-agent/pkg/util/scrubber"
3334

3435
dto "github.com/prometheus/client_model/go"
@@ -76,6 +77,13 @@ type Provides struct {
7677
}
7778

7879
// Interfacing with runner.
80+
//
81+
// A single job type drives both the periodic metric-profile flush and
82+
// the errortracking flush. The profiles slice doubles as a
83+
// discriminator: a nil profiles slice means "this is the errortracking
84+
// flush job"; a non-nil slice means "this is a metric-profile tick".
85+
// Threading both behaviours through the same job avoids widening the
86+
// runner's interface for a one-off second consumer.
7987
type job struct {
8088
a *atel
8189
profiles []*Profile
@@ -634,6 +642,10 @@ func (a *atel) SendEvent(eventType string, eventPayload []byte) error {
634642
return nil
635643
}
636644

645+
// SubmitErrorLog is a no-op in stack-2; the actual channel send and
646+
// flush machinery lives in stack-3 (wiring).
647+
func (a *atel) SubmitErrorLog(_ errortracking.ErrorLog) {}
648+
637649
func (a *atel) StartStartupSpan(operationName string) (*installertelemetry.Span, context.Context) {
638650
if a.lightTracer != nil {
639651
return installertelemetry.StartSpanFromContext(a.cancelCtx, operationName)
@@ -694,7 +706,6 @@ func (a *atel) stop() error {
694706
runnerCtx := a.runner.stop()
695707
<-runnerCtx.Done()
696708

697-
<-a.cancelCtx.Done()
698709
a.logComp.Info("Agent telemetry is stopped")
699710
return nil
700711
}

comp/core/agenttelemetry/impl/agenttelemetry_test.go

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"maps"
1515
"net/http"
1616
"strings"
17+
"sync"
1718
"testing"
1819

1920
dto "github.com/prometheus/client_model/go"
@@ -53,6 +54,21 @@ func newClientMock() client {
5354
// Sender mock
5455
type senderMock struct {
5556
sentMetrics []*agentmetric
57+
58+
// Captures from the errortracking flush path. Protected by mu
59+
// because the flush job may run concurrently with test setup and
60+
// assertions; readers MUST take the lock or use a synchronisation
61+
// barrier (e.g. wait on runner.stop().Done) that establishes
62+
// happens-before with the job's completion.
63+
//
64+
// sendLogsCallCount counts sendLogsBatch invocations; sentLogs
65+
// flattens every batch into one accumulating slice. The pair lets
66+
// tests distinguish "1 call with N records" from "N calls with 1
67+
// record each" — the latter would be a regression to per-batch
68+
// dispatch that the flattened slice alone cannot detect.
69+
sentLogsMu sync.Mutex
70+
sentLogs []Log
71+
sendLogsCallCount int
5672
}
5773

5874
func (s *senderMock) startSession(_ context.Context) *senderSession {
@@ -66,6 +82,34 @@ func (s *senderMock) sendAgentMetricPayloads(_ *senderSession, metrics []*agentm
6682
}
6783
func (s *senderMock) sendEventPayload(_ *senderSession, _ *Event, _ map[string]interface{}) {
6884
}
85+
func (s *senderMock) sendLogsBatch(_ context.Context, logs []Log) error {
86+
s.sentLogsMu.Lock()
87+
defer s.sentLogsMu.Unlock()
88+
s.sendLogsCallCount++
89+
s.sentLogs = append(s.sentLogs, logs...)
90+
return nil
91+
}
92+
93+
// capturedLogs returns a thread-safe snapshot of the records captured
94+
// via sendLogsBatch. Tests should call this rather than reading
95+
// sentLogs directly.
96+
func (s *senderMock) capturedLogs() []Log {
97+
s.sentLogsMu.Lock()
98+
defer s.sentLogsMu.Unlock()
99+
out := make([]Log, len(s.sentLogs))
100+
copy(out, s.sentLogs)
101+
return out
102+
}
103+
104+
// sendLogsCalls returns a thread-safe snapshot of how many times
105+
// sendLogsBatch was invoked. Pair with capturedLogs to assert
106+
// "one HTTP call per flush" (N records via 1 call, not 1 record via N
107+
// calls).
108+
func (s *senderMock) sendLogsCalls() int {
109+
s.sentLogsMu.Lock()
110+
defer s.sentLogsMu.Unlock()
111+
return s.sendLogsCallCount
112+
}
69113

70114
// Runner mock (TODO: use use mock.Mock)
71115
type runnerMock struct {
@@ -75,7 +119,7 @@ type runnerMock struct {
75119

76120
func (r *runnerMock) run() {
77121
for _, j := range r.jobs {
78-
j.a.run(j.profiles)
122+
j.Run()
79123
}
80124
}
81125

@@ -1171,6 +1215,25 @@ func TestSenderConfigDDUrlWithEmptyAdditionalPoint(t *testing.T) {
11711215
assert.Equal(t, "https://instrumentation-telemetry-intake.us5.datadoghq.com./api/v2/apmtelemetry", url)
11721216
}
11731217

1218+
// TestSenderConfigLogsNoSSL verifies that logs_no_ssl: true causes buildURL to
1219+
// produce an http:// URL. Previously buildURL hardcoded "https" and ignored
1220+
// Endpoint.UseSSL(), silently dropping all telemetry in no-SSL environments.
1221+
func TestSenderConfigLogsNoSSL(t *testing.T) {
1222+
c := `
1223+
api_key: foo
1224+
agent_telemetry:
1225+
enabled: true
1226+
logs_dd_url: "localhost:19999"
1227+
logs_no_ssl: true
1228+
`
1229+
sndr := makeSenderImpl(t, nil, c)
1230+
assert.NotNil(t, sndr)
1231+
1232+
assert.Len(t, sndr.(*senderImpl).endpoints.Endpoints, 1)
1233+
url := buildURL(sndr.(*senderImpl).endpoints.Endpoints[0])
1234+
assert.Equal(t, "http://localhost:19999/api/v2/apmtelemetry", url)
1235+
}
1236+
11741237
func TestGetAsJSONScrub(t *testing.T) {
11751238
var c = `
11761239
agent_telemetry:

comp/core/agenttelemetry/impl/go.mod

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,15 @@ require (
1818
github.com/DataDog/datadog-agent/pkg/util/hostinfo v0.0.0-20251027120702-0e91eee9852f
1919
github.com/DataDog/datadog-agent/pkg/util/http v0.75.4
2020
github.com/DataDog/datadog-agent/pkg/util/jsonquery v0.0.0-20251027120702-0e91eee9852f
21+
github.com/DataDog/datadog-agent/pkg/util/log v0.75.4
22+
github.com/DataDog/datadog-agent/pkg/util/log/setup v0.0.0-00010101000000-000000000000
2123
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.75.4
2224
github.com/DataDog/datadog-agent/pkg/version v0.75.4
2325
github.com/DataDog/zstd v1.5.8-0.20260421145859-31a7e515a571
2426
github.com/prometheus/client_model v0.6.2
2527
github.com/robfig/cron/v3 v3.0.1
2628
github.com/stretchr/testify v1.11.1
29+
go.uber.org/atomic v1.11.0
2730
go.yaml.in/yaml/v2 v2.4.4
2831
)
2932

@@ -54,7 +57,6 @@ require (
5457
github.com/DataDog/datadog-agent/pkg/util/defaultpaths v0.64.0-devel // indirect
5558
github.com/DataDog/datadog-agent/pkg/util/executable v0.75.4 // indirect
5659
github.com/DataDog/datadog-agent/pkg/util/filesystem v0.75.4 // indirect
57-
github.com/DataDog/datadog-agent/pkg/util/log v0.75.4 // indirect
5860
github.com/DataDog/datadog-agent/pkg/util/option v0.75.4 // indirect
5961
github.com/DataDog/datadog-agent/pkg/util/pointer v0.75.4 // indirect
6062
github.com/DataDog/datadog-agent/pkg/util/system v0.75.4 // indirect
@@ -92,7 +94,6 @@ require (
9294
github.com/tklauser/go-sysconf v0.3.16 // indirect
9395
github.com/tklauser/numcpus v0.11.0 // indirect
9496
github.com/yusufpapurcu/wmi v1.2.4 // indirect
95-
go.uber.org/atomic v1.11.0 // indirect
9697
go.uber.org/dig v1.19.0 // indirect
9798
go.uber.org/fx v1.24.0 // indirect
9899
go.uber.org/multierr v1.11.0 // indirect

0 commit comments

Comments
 (0)