Skip to content

Commit 976ed10

Browse files
khanayan123claude
andcommitted
Implement app-extended-heartbeat telemetry event
Add support for app-extended-heartbeat telemetry event that sends configuration, dependencies, and integrations data at a configurable interval (default: 24 hours). This event serves as a failsafe for reconstructing application records in case of data failures. Changes: - Add DD_TELEMETRY_EXTENDED_HEARTBEAT_INTERVAL environment variable - Add extended_heartbeat_interval configuration to Configuration and FinalizedConfiguration structs - Implement app_extended_heartbeat_payload() method that generates payload with configuration, dependencies, and integrations arrays - Schedule recurring extended heartbeat task alongside regular heartbeat - Add comprehensive test coverage for extended heartbeat functionality - Fix existing test bug: check heartbeat_interval instead of metrics_interval The payload excludes 'products' and 'install_signature' fields as per the API specification, focusing only on configuration, dependencies, and integrations data. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 32898b5 commit 976ed10

6 files changed

Lines changed: 134 additions & 1 deletion

File tree

include/datadog/environment.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ namespace environment {
5151
MACRO(DD_VERSION) \
5252
MACRO(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) \
5353
MACRO(DD_TELEMETRY_HEARTBEAT_INTERVAL) \
54+
MACRO(DD_TELEMETRY_EXTENDED_HEARTBEAT_INTERVAL) \
5455
MACRO(DD_TELEMETRY_METRICS_ENABLED) \
5556
MACRO(DD_TELEMETRY_METRICS_INTERVAL_SECONDS) \
5657
MACRO(DD_TELEMETRY_DEBUG) \

include/datadog/telemetry/configuration.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ struct Configuration {
2929
// Interval at which the heartbeat payload will be sent.
3030
// Can be overriden by `DD_TELEMETRY_HEARTBEAT_INTERVAL` environment variable.
3131
tracing::Optional<double> heartbeat_interval_seconds;
32+
// Interval at which the extended heartbeat payload will be sent.
33+
// Can be overriden by `DD_TELEMETRY_EXTENDED_HEARTBEAT_INTERVAL` environment variable.
34+
tracing::Optional<double> extended_heartbeat_interval_seconds;
3235
// `integration_name` is the name of the product integrating this library.
3336
// Example: "nginx", "envoy" or "istio".
3437
tracing::Optional<std::string> integration_name;
@@ -52,6 +55,7 @@ struct FinalizedConfiguration {
5255
bool report_logs;
5356
std::chrono::steady_clock::duration metrics_interval;
5457
std::chrono::steady_clock::duration heartbeat_interval;
58+
std::chrono::steady_clock::duration extended_heartbeat_interval;
5559
std::string integration_name;
5660
std::string integration_version;
5761
std::vector<Product> products;

src/datadog/telemetry/configuration.cpp

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,16 @@ tracing::Expected<Configuration> load_telemetry_env_config() {
4848
env_cfg.heartbeat_interval_seconds = *maybe_value;
4949
}
5050

51+
if (auto extended_heartbeat_interval_seconds =
52+
lookup(environment::DD_TELEMETRY_EXTENDED_HEARTBEAT_INTERVAL)) {
53+
auto maybe_value = parse_double(*extended_heartbeat_interval_seconds);
54+
if (auto error = maybe_value.if_error()) {
55+
return *error;
56+
}
57+
58+
env_cfg.extended_heartbeat_interval_seconds = *maybe_value;
59+
}
60+
5161
return env_cfg;
5262
}
5363

@@ -112,6 +122,19 @@ tracing::Expected<FinalizedConfiguration> finalize_config(
112122
std::chrono::duration_cast<std::chrono::milliseconds>(
113123
std::chrono::duration<double>(heartbeat_interval.second));
114124

125+
// extended_heartbeat_interval_seconds
126+
auto extended_heartbeat_interval =
127+
pick(env_config->extended_heartbeat_interval_seconds,
128+
user_config.extended_heartbeat_interval_seconds, 86400);
129+
if (extended_heartbeat_interval.second <= 0.) {
130+
return Error{Error::Code::OUT_OF_RANGE_INTEGER,
131+
"Telemetry extended heartbeat polling interval must be a "
132+
"positive value"};
133+
}
134+
result.extended_heartbeat_interval =
135+
std::chrono::duration_cast<std::chrono::milliseconds>(
136+
std::chrono::duration<double>(extended_heartbeat_interval.second));
137+
115138
// integration_name
116139
std::tie(origin, result.integration_name) =
117140
pick(env_config->integration_name, user_config.integration_name,

src/datadog/telemetry/telemetry_impl.cpp

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,12 @@ void Telemetry::schedule_tasks() {
223223
config_.heartbeat_interval,
224224
[this]() { send_payload("app-heartbeat", heartbeat_and_telemetry()); }));
225225

226+
tasks_.emplace_back(scheduler_->schedule_recurring_event(
227+
config_.extended_heartbeat_interval, [this]() {
228+
send_payload("app-extended-heartbeat",
229+
app_extended_heartbeat_payload());
230+
}));
231+
226232
if (config_.report_metrics) {
227233
tasks_.emplace_back(scheduler_->schedule_recurring_event(
228234
config_.metrics_interval, [this]() mutable { capture_metrics(); }));
@@ -678,6 +684,40 @@ std::string Telemetry::app_started_payload() {
678684
return batch.dump();
679685
}
680686

687+
std::string Telemetry::app_extended_heartbeat_payload() {
688+
auto configuration_json = nlohmann::json::array();
689+
690+
for (const auto& product : config_.products) {
691+
auto& configurations = product.configurations;
692+
for (const auto& [_, config_metadatas] : configurations) {
693+
for (const auto& config_metadata : config_metadatas) {
694+
configuration_json.emplace_back(
695+
generate_configuration_field(config_metadata));
696+
}
697+
}
698+
}
699+
700+
auto dependencies_json = nlohmann::json::array();
701+
702+
auto integrations_json = nlohmann::json::array();
703+
if (!config_.integration_name.empty()) {
704+
integrations_json.emplace_back(nlohmann::json{
705+
{"name", config_.integration_name},
706+
{"version", config_.integration_version},
707+
{"enabled", true},
708+
});
709+
}
710+
711+
auto telemetry_body = generate_telemetry_body("app-extended-heartbeat");
712+
telemetry_body["payload"] = nlohmann::json{
713+
{"configuration", configuration_json},
714+
{"dependencies", dependencies_json},
715+
{"integrations", integrations_json},
716+
};
717+
718+
return telemetry_body.dump();
719+
}
720+
681721
nlohmann::json Telemetry::generate_telemetry_body(std::string request_type) {
682722
std::time_t tracer_time = std::chrono::duration_cast<std::chrono::seconds>(
683723
clock_().wall.time_since_epoch())

src/datadog/telemetry/telemetry_impl.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,10 @@ class Telemetry final {
152152
// Constructs a messsage-batch containing `app-heartbeat`, and if metrics
153153
// have been modified, a `generate-metrics` message.
154154
std::string heartbeat_and_telemetry();
155+
// Constructs an `app-extended-heartbeat` message containing configuration,
156+
// dependencies, and integrations (reuses app-started payload logic but
157+
// excludes products and install_signature).
158+
std::string app_extended_heartbeat_payload();
155159
// Constructs a message-batch containing `app-closing`, and if metrics have
156160
// been modified, a `generate-metrics` message.
157161
std::string app_closing_payload();

test/telemetry/test_telemetry.cpp

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,10 @@ std::optional<nlohmann::json> find_payload(const nlohmann::json& messages,
4545
struct FakeEventScheduler : public EventScheduler {
4646
size_t count_tasks = 0;
4747
std::function<void()> heartbeat_callback = nullptr;
48+
std::function<void()> extended_heartbeat_callback = nullptr;
4849
std::function<void()> metrics_callback = nullptr;
4950
Optional<std::chrono::steady_clock::duration> heartbeat_interval;
51+
Optional<std::chrono::steady_clock::duration> extended_heartbeat_interval;
5052
Optional<std::chrono::steady_clock::duration> metrics_interval;
5153
bool cancelled = false;
5254

@@ -57,6 +59,9 @@ struct FakeEventScheduler : public EventScheduler {
5759
heartbeat_callback = callback;
5860
heartbeat_interval = interval;
5961
} else if (count_tasks == 1) {
62+
extended_heartbeat_callback = callback;
63+
extended_heartbeat_interval = interval;
64+
} else if (count_tasks == 2) {
6065
metrics_callback = callback;
6166
metrics_interval = interval;
6267
}
@@ -69,6 +74,11 @@ struct FakeEventScheduler : public EventScheduler {
6974
heartbeat_callback();
7075
}
7176

77+
void trigger_extended_heartbeat() {
78+
assert(extended_heartbeat_callback != nullptr);
79+
extended_heartbeat_callback();
80+
}
81+
7282
void trigger_metrics_capture() {
7383
assert(metrics_callback != nullptr);
7484
metrics_callback();
@@ -391,6 +401,54 @@ TELEMETRY_IMPLEMENTATION_TEST("Tracer telemetry API") {
391401
REQUIRE(find_payload(message_batch["payload"], "app-heartbeat"));
392402
}
393403

404+
SECTION("generates an extended heartbeat message") {
405+
client->clear();
406+
407+
Configuration cfg;
408+
cfg.extended_heartbeat_interval_seconds = 1;
409+
Product product;
410+
product.name = Product::Name::tracing;
411+
product.enabled = true;
412+
product.version = tracer_version;
413+
product.configurations =
414+
std::unordered_map<ConfigName, std::vector<ConfigMetadata>>{
415+
{ConfigName::SERVICE_NAME,
416+
{ConfigMetadata(ConfigName::SERVICE_NAME, "test-service",
417+
ConfigMetadata::Origin::CODE)}},
418+
};
419+
cfg.products.emplace_back(std::move(product));
420+
421+
auto final_cfg = finalize_config(cfg);
422+
REQUIRE(final_cfg);
423+
424+
Telemetry telemetry_extended{*final_cfg, tracer_signature, logger, client,
425+
scheduler, *url, clock};
426+
427+
client->clear();
428+
auto extended_heartbeat_message =
429+
telemetry_extended.app_extended_heartbeat_payload();
430+
auto extended_heartbeat =
431+
nlohmann::json::parse(extended_heartbeat_message);
432+
433+
REQUIRE(is_valid_telemetry_payload(extended_heartbeat) == true);
434+
REQUIRE(extended_heartbeat["request_type"] == "app-extended-heartbeat");
435+
REQUIRE(extended_heartbeat["payload"].contains("configuration"));
436+
REQUIRE(extended_heartbeat["payload"].contains("dependencies"));
437+
REQUIRE(extended_heartbeat["payload"].contains("integrations"));
438+
439+
REQUIRE(extended_heartbeat["payload"]["configuration"].is_array());
440+
REQUIRE(extended_heartbeat["payload"]["dependencies"].is_array());
441+
REQUIRE(extended_heartbeat["payload"]["integrations"].is_array());
442+
443+
CHECK(!extended_heartbeat["payload"].contains("products"));
444+
CHECK(!extended_heartbeat["payload"].contains("install_signature"));
445+
446+
auto cfg_payload = extended_heartbeat["payload"]["configuration"];
447+
REQUIRE(cfg_payload.size() == 1);
448+
CHECK(cfg_payload[0]["name"] == "service");
449+
CHECK(cfg_payload[0]["value"] == "test-service");
450+
}
451+
394452
SECTION("metrics reporting") {
395453
SECTION("counters are correctly serialized in generate-metrics payload") {
396454
client->clear();
@@ -903,7 +961,10 @@ TELEMETRY_IMPLEMENTATION_TEST("Tracer telemetry configuration") {
903961
CHECK(scheduler->metrics_interval == 500ms);
904962

905963
CHECK(scheduler->heartbeat_callback != nullptr);
906-
CHECK(scheduler->metrics_interval != 30s);
964+
CHECK(scheduler->heartbeat_interval == 30s);
965+
966+
CHECK(scheduler->extended_heartbeat_callback != nullptr);
967+
CHECK(scheduler->extended_heartbeat_interval == 86400s);
907968
}
908969

909970
SECTION("disabling logs reporting do not collect logs") {

0 commit comments

Comments
 (0)