Skip to content

Commit c485d0c

Browse files
pducolinpducolin
authored
feat(errortracking): wire error-tracking handler and add E2E tests (#52488)
> **Stacked PR 4/4** — part of [#50607](#50607). Builds on [stack-3 #52487](#52487). ### What does this PR do? Wires the error-tracking handler into the agent and validates the full pipeline end-to-end: - **Config defaults** — adds `buffer_size`, `flush_interval`, `bouncer_window` to `common_settings.go` - **Log setup** — wires `errortracking.Handler` into `buildSlogLogger` as a synchronous sibling handler - **Agent run command** — installs an `fx.Lifecycle` hook that starts/stops the handler - **E2E test suite** (`test/new-e2e/tests/agent-telemetry/`) — provisions a real AWS EC2 VM via Pulumi, installs the agent with a Python check that logs errors and raises an exception. Asserts that error log payloads reach the internal telemetry intake. Tests both `errortracking.enabled: true` and the default disabled state. - **CI** — extends `.on_agenttelemetry_or_e2e_changes` with `test/new-e2e/tests/agent-telemetry/**/*` ### Stack 1. [stack-1 #52485] Core `pkg/util/log/errortracking/` package 2. [stack-2 #52486] agenttelemetry component extensions + sender refactor 3. [stack-3 #52487] FakeIntake aggregator/client + CI scaffolding 4. **This PR** — agent wiring + E2E test Co-authored-by: paola.ducolin <paola.ducolin@datadoghq.com>
1 parent 1e5e13c commit c485d0c

27 files changed

Lines changed: 1716 additions & 52 deletions

File tree

‎.gitlab-ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1389,8 +1389,11 @@ workflow:
13891389
- !reference [.on_e2e_main_release_or_rc]
13901390
- changes:
13911391
paths:
1392+
- cmd/agent/subcommands/run/**/*
13921393
- comp/core/agenttelemetry/**/*
1394+
- pkg/util/log/setup/**/*
13931395
- pkg/util/log/errortracking/**/*
1396+
- test/new-e2e/tests/agent-telemetry/**/*
13941397
compare_to: $COMPARE_TO_BRANCH
13951398
when: on_success
13961399

‎.gitlab/test/e2e/e2e.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -437,8 +437,7 @@ new-e2e-agent-health:
437437
new-e2e-agent-telemetry:
438438
extends: .new_e2e_template_needs_deb_x64
439439
rules:
440-
# only manual, will enable on changes in follow up pr
441-
# - !reference [.on_agenttelemetry_or_e2e_changes]
440+
- !reference [.on_agenttelemetry_or_e2e_changes]
442441
- !reference [.manual]
443442
variables:
444443
TARGETS: ./tests/agent-telemetry

‎BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,7 @@ exports_files(["go.mod"])
333333
# gazelle:exclude test/new-e2e/tests/agent-subcommands/check
334334
# gazelle:exclude test/new-e2e/tests/agent-subcommands/dogstatsdreplay
335335
# gazelle:exclude test/new-e2e/tests/agent-subcommands/status
336+
# gazelle:exclude test/new-e2e/tests/agent-telemetry
336337
# gazelle:exclude test/new-e2e/tests/anomalydetection
337338
# gazelle:exclude test/new-e2e/tests/apm
338339
# gazelle:exclude test/new-e2e/tests/autoscaling/spot

‎cmd/agent/subcommands/run/command.go‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,8 @@ import (
205205
"github.com/DataDog/datadog-agent/pkg/util/installinfo"
206206
"github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver/leaderelection"
207207
pkglog "github.com/DataDog/datadog-agent/pkg/util/log"
208+
errortrackingpkg "github.com/DataDog/datadog-agent/pkg/util/log/errortracking"
209+
pkglogsetup "github.com/DataDog/datadog-agent/pkg/util/log/setup"
208210
"github.com/DataDog/datadog-agent/pkg/util/option"
209211
"github.com/DataDog/datadog-agent/pkg/version"
210212

@@ -403,7 +405,7 @@ func run(log log.Component,
403405
"Establish if the agent is running",
404406
)
405407

406-
// agentStarted and agentRunning are metrics used for Cross-org Agent Telemetry (COAT)
408+
// agentStarted and agentRunning are metrics used for internal agent telemetry
407409
// for more details on the scheduling config check comp/core/agenttelemetry/impl/config.go
408410
agentStarted.Inc()
409411
agentRunning.Set(1)
@@ -579,6 +581,8 @@ func getSharedFxOption() fx.Option {
579581
}),
580582
settingsfx.Module(),
581583
agenttelemetryfx.Module(),
584+
// errortracking submitter wire — atel owns buffer/flush/recursion.
585+
fx.Invoke(installErrortrackingHandler),
582586
remotetraceroute.Module(),
583587
networkpath.Bundle(),
584588
syntheticsTestsfx.Module(),
@@ -599,6 +603,33 @@ func getSharedFxOption() fx.Option {
599603
)
600604
}
601605

606+
// installErrortrackingHandler is a no-op when the feature is disabled
607+
// (agent_telemetry.errortracking.enabled or the parent agent_telemetry
608+
// gate). The OnStart hook installs the submitter into pkg/util/log/setup;
609+
// the matching clear runs synchronously inside atel.stop()
610+
// (deliberately not as a separate OnStop hook here) so it precedes the
611+
// final flush-goroutine drain.
612+
func installErrortrackingHandler(lc fx.Lifecycle, cfg config.Component, at agenttelemetry.Component) {
613+
if !configUtils.IsErrorTrackingEnabled(cfg) {
614+
return
615+
}
616+
617+
submitter := func(elog errortrackingpkg.ErrorLog) {
618+
at.SubmitErrorLog(elog)
619+
}
620+
621+
bouncerWindow := time.Duration(cfg.GetInt("agent_telemetry.errortracking.bouncer_window_seconds")) * time.Second
622+
bouncer := errortrackingpkg.NewBouncer(bouncerWindow, 0)
623+
624+
lc.Append(fx.Hook{
625+
OnStart: func(_ context.Context) error {
626+
pkglogsetup.RegisterErrortrackingSubmitter(submitter)
627+
pkglogsetup.RegisterErrortrackingBouncer(bouncer)
628+
return nil
629+
},
630+
})
631+
}
632+
602633
// startAgent Initializes the agent process
603634
func startAgent(
604635
log log.Component,

‎comp/core/agenttelemetry/fx/go.mod‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,43 +18,44 @@ require (
1818
github.com/DataDog/datadog-agent/comp/core/flare/builder v0.70.0 // indirect
1919
github.com/DataDog/datadog-agent/comp/core/flare/types v0.72.0-rc.1 // indirect
2020
github.com/DataDog/datadog-agent/comp/core/log/def v0.70.0 // indirect
21-
github.com/DataDog/datadog-agent/comp/core/secrets/def v0.75.4 // indirect
21+
github.com/DataDog/datadog-agent/comp/core/secrets/def v0.77.0 // indirect
2222
github.com/DataDog/datadog-agent/comp/core/secrets/noop-impl v0.77.0-devel.0.20260211235139-a5361978c2b6 // indirect
2323
github.com/DataDog/datadog-agent/comp/core/telemetry v0.75.4 // indirect
2424
github.com/DataDog/datadog-agent/comp/def v0.75.4 // indirect
2525
github.com/DataDog/datadog-agent/comp/logs-library v0.0.0-00010101000000-000000000000 // indirect
2626
github.com/DataDog/datadog-agent/comp/logs/agent/config v0.75.4 // indirect
27-
github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.75.4 // indirect
28-
github.com/DataDog/datadog-agent/pkg/config/basic v0.0.0-20260211235139-a5361978c2b6 // indirect
27+
github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.77.0 // indirect
28+
github.com/DataDog/datadog-agent/pkg/config/basic v0.77.0 // indirect
2929
github.com/DataDog/datadog-agent/pkg/config/buildschema v0.0.0-00010101000000-000000000000 // indirect
30-
github.com/DataDog/datadog-agent/pkg/config/create v0.75.4 // indirect
31-
github.com/DataDog/datadog-agent/pkg/config/env v0.75.4 // indirect
32-
github.com/DataDog/datadog-agent/pkg/config/helper v0.75.4 // indirect
33-
github.com/DataDog/datadog-agent/pkg/config/mock v0.75.4 // indirect
30+
github.com/DataDog/datadog-agent/pkg/config/create v0.77.0 // indirect
31+
github.com/DataDog/datadog-agent/pkg/config/env v0.77.0 // indirect
32+
github.com/DataDog/datadog-agent/pkg/config/helper v0.77.0 // indirect
33+
github.com/DataDog/datadog-agent/pkg/config/mock v0.77.0 // indirect
3434
github.com/DataDog/datadog-agent/pkg/config/model v0.77.2 // indirect
35-
github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.75.4 // indirect
36-
github.com/DataDog/datadog-agent/pkg/config/setup v0.75.4 // indirect
37-
github.com/DataDog/datadog-agent/pkg/config/structure v0.77.0-devel.0.20260211235139-a5361978c2b6 // indirect
35+
github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.77.0 // indirect
36+
github.com/DataDog/datadog-agent/pkg/config/setup v0.77.0 // indirect
37+
github.com/DataDog/datadog-agent/pkg/config/structure v0.77.0 // indirect
3838
github.com/DataDog/datadog-agent/pkg/config/utils v0.75.4 // indirect
39-
github.com/DataDog/datadog-agent/pkg/fips v0.75.4 // indirect
39+
github.com/DataDog/datadog-agent/pkg/fips v0.77.0 // indirect
4040
github.com/DataDog/datadog-agent/pkg/fleet/installer v0.70.0 // indirect
4141
github.com/DataDog/datadog-agent/pkg/gohai v0.69.4 // indirect
4242
github.com/DataDog/datadog-agent/pkg/logs/types v0.75.4 // indirect
43-
github.com/DataDog/datadog-agent/pkg/template v0.75.4 // indirect
43+
github.com/DataDog/datadog-agent/pkg/template v0.77.0 // indirect
4444
github.com/DataDog/datadog-agent/pkg/util/cache v0.69.4 // indirect
4545
github.com/DataDog/datadog-agent/pkg/util/defaultpaths v0.64.0-devel // indirect
46-
github.com/DataDog/datadog-agent/pkg/util/executable v0.75.4 // indirect
47-
github.com/DataDog/datadog-agent/pkg/util/filesystem v0.75.4 // indirect
46+
github.com/DataDog/datadog-agent/pkg/util/executable v0.77.0 // indirect
47+
github.com/DataDog/datadog-agent/pkg/util/filesystem v0.77.0 // indirect
4848
github.com/DataDog/datadog-agent/pkg/util/hostinfo v0.0.0-20251027120702-0e91eee9852f // indirect
4949
github.com/DataDog/datadog-agent/pkg/util/http v0.75.4 // indirect
50-
github.com/DataDog/datadog-agent/pkg/util/log v0.75.4 // indirect
50+
github.com/DataDog/datadog-agent/pkg/util/log v0.77.0 // indirect
51+
github.com/DataDog/datadog-agent/pkg/util/log/setup v0.77.0 // indirect
5152
github.com/DataDog/datadog-agent/pkg/util/option v0.75.4 // indirect
52-
github.com/DataDog/datadog-agent/pkg/util/pointer v0.75.4 // indirect
53-
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.75.4 // indirect
54-
github.com/DataDog/datadog-agent/pkg/util/system v0.75.4 // indirect
53+
github.com/DataDog/datadog-agent/pkg/util/pointer v0.77.0 // indirect
54+
github.com/DataDog/datadog-agent/pkg/util/scrubber v0.77.0 // indirect
55+
github.com/DataDog/datadog-agent/pkg/util/system v0.77.0 // indirect
5556
github.com/DataDog/datadog-agent/pkg/util/uuid v0.69.4 // indirect
56-
github.com/DataDog/datadog-agent/pkg/util/winutil v0.75.4 // indirect
57-
github.com/DataDog/datadog-agent/pkg/version v0.75.4 // indirect
57+
github.com/DataDog/datadog-agent/pkg/util/winutil v0.77.0 // indirect
58+
github.com/DataDog/datadog-agent/pkg/version v0.77.0 // indirect
5859
github.com/DataDog/go-acl v1.0.1 // indirect
5960
github.com/DataDog/zstd v1.5.8-0.20260421145859-31a7e515a571 // indirect
6061
github.com/Microsoft/go-winio v0.6.2 // indirect

‎comp/core/agenttelemetry/impl/BUILD.bazel‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ go_library(
66
srcs = [
77
"agenttelemetry.go",
88
"config.go",
9+
"errortracking_sender.go",
10+
"logs_payload.go",
911
"runner.go",
1012
"sender.go",
1113
"utils.go",
@@ -26,31 +28,39 @@ go_library(
2628
"//pkg/util/hostinfo",
2729
"//pkg/util/http",
2830
"//pkg/util/log/errortracking",
31+
"//pkg/util/log/setup",
2932
"//pkg/util/scrubber",
3033
"//pkg/version",
3134
"@com_github_datadog_zstd//:zstd",
3235
"@com_github_prometheus_client_model//go",
3336
"@com_github_robfig_cron_v3//:cron",
3437
"@in_yaml_go_yaml_v2//:yaml",
38+
"@org_uber_go_atomic//:atomic",
3539
],
3640
)
3741

3842
dd_agent_go_test(
3943
name = "impl_test",
40-
srcs = ["agenttelemetry_test.go"],
44+
srcs = [
45+
"agenttelemetry_test.go",
46+
"errortracking_sender_test.go",
47+
],
4148
embed = [":impl"],
4249
deps = [
4350
"//comp/core/log/def",
4451
"//comp/core/log/mock",
4552
"//comp/core/telemetry/def",
4653
"//comp/core/telemetry/mock",
54+
"//comp/logs/agent/config",
4755
"//pkg/config/mock",
4856
"//pkg/util/fxutil",
4957
"//pkg/util/jsonquery",
58+
"//pkg/util/log/errortracking",
5059
"@com_github_datadog_zstd//:zstd",
5160
"@com_github_prometheus_client_model//go",
5261
"@com_github_stretchr_testify//assert",
5362
"@com_github_stretchr_testify//mock",
5463
"@com_github_stretchr_testify//require",
64+
"@org_uber_go_atomic//:atomic",
5565
],
5666
)

0 commit comments

Comments
 (0)