Skip to content

Commit 674cd35

Browse files
duncanistaduncanista
authored
chore(deps): upgrade dogstatsd (#1020)
## Overview Continuation of #1018 removing unnecessary mut lock on callers for dogstatsd
1 parent 7637378 commit 674cd35

9 files changed

Lines changed: 458 additions & 142 deletions

File tree

‎bottlecap/Cargo.lock‎

Lines changed: 230 additions & 65 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bottlecap/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,8 @@ libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "158b
7171
libdd-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "158b59471f1132e3cb36023fa3c46ccb2dd0eda1" }
7272
libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "158b59471f1132e3cb36023fa3c46ccb2dd0eda1" }
7373
libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog", rev = "158b59471f1132e3cb36023fa3c46ccb2dd0eda1" }
74-
dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "18b49baba8bfef97060d7edd8b830584d0da3373", default-features = false }
75-
datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "18b49baba8bfef97060d7edd8b830584d0da3373", default-features = false }
74+
dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "2eb009a59ed07ffcaf174db1c31af413365e9bc6", default-features = false }
75+
datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "2eb009a59ed07ffcaf174db1c31af413365e9bc6", default-features = false }
7676
libddwaf = { version = "1.28.1", git = "https://github.com/DataDog/libddwaf-rust", rev = "d1534a158d976bd4f747bf9fcc58e0712d2d17fc", default-features = false, features = ["serde"] }
7777

7878
[dev-dependencies]

‎bottlecap/LICENSE-3rdparty.csv‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ h2,https://github.com/hyperium/h2,MIT,"Carl Lerche <me@carllerche.com>, Sean McA
6666
hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,Amanieu d'Antras <amanieu@gmail.com>
6767
headers,https://github.com/hyperium/headers,MIT,Sean McArthur <sean@seanmonstar.com>
6868
headers-core,https://github.com/hyperium/headers,MIT,Sean McArthur <sean@seanmonstar.com>
69+
heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors
6970
hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi <kokakiwi@kokakiwi.net>
7071
hmac,https://github.com/RustCrypto/MACs,MIT OR Apache-2.0,RustCrypto Developers
7172
http,https://github.com/hyperium/http,MIT OR Apache-2.0,"Alex Crichton <alex@alexcrichton.com>, Carl Lerche <me@carllerche.com>, Sean McArthur <sean@seanmonstar.com>"
@@ -85,6 +86,7 @@ icu_normalizer_data,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X P
8586
icu_properties,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers
8687
icu_properties_data,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers
8788
icu_provider,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers
89+
id-arena,https://github.com/fitzgen/id-arena,MIT OR Apache-2.0,"Nick Fitzgerald <fitzgen@gmail.com>, Aleksey Kladov <aleksey.kladov@gmail.com>"
8890
idna,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers
8991
idna_adapter,https://github.com/hsivonen/idna_adapter,Apache-2.0 OR MIT,The rust-url developers
9092
indexmap,https://github.com/bluss/indexmap,Apache-2.0 OR MIT,The indexmap Authors
@@ -97,6 +99,7 @@ itoa,https://github.com/dtolnay/itoa,MIT OR Apache-2.0,David Tolnay <dtolnay@gma
9799
jobserver,https://github.com/rust-lang/jobserver-rs,MIT OR Apache-2.0,Alex Crichton <alex@alexcrichton.com>
98100
js-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers
99101
lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel <loebel.marvin@gmail.com>
102+
leb128fmt,https://github.com/bluk/leb128fmt,MIT OR Apache-2.0,Bryant Luk <code@bryantluk.com>
100103
libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers
101104
libdd-common,https://github.com/DataDog/libdatadog/tree/main/datadog-common,Apache-2.0,The libdd-common Authors
102105
libdd-ddsketch,https://github.com/DataDog/libdatadog/tree/main/libdd-ddsketch,Apache-2.0,The libdd-ddsketch Authors
@@ -144,6 +147,7 @@ pin-utils,https://github.com/rust-lang-nursery/pin-utils,MIT OR Apache-2.0,Josef
144147
potential_utf,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers
145148
powerfmt,https://github.com/jhpratt/powerfmt,MIT OR Apache-2.0,Jacob Pratt <jacob@jhpratt.dev>
146149
ppv-lite86,https://github.com/cryptocorrosion/cryptocorrosion,MIT OR Apache-2.0,The CryptoCorrosion Contributors
150+
prettyplease,https://github.com/dtolnay/prettyplease,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
147151
proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay <dtolnay@gmail.com>, Alex Crichton <alex@alexcrichton.com>"
148152
proc-macro2-diagnostics,https://github.com/SergioBenitez/proc-macro2-diagnostics,MIT OR Apache-2.0,Sergio Benitez <sb@sergio.bz>
149153
prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert <dan@danburkert.com>, Lucio Franco <luciofranco14@gmail.com>, Casper Meijn <casper@meijn.net>, Tokio Contributors <team@tokio.rs>"
@@ -182,6 +186,7 @@ schannel,https://github.com/steffengy/schannel-rs,MIT,"Steven Fackler <sfackler@
182186
scopeguard,https://github.com/bluss/scopeguard,MIT OR Apache-2.0,bluss
183187
security-framework,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler <sfackler@gmail.com>, Kornel <kornel@geekhood.net>"
184188
security-framework-sys,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler <sfackler@gmail.com>, Kornel <kornel@geekhood.net>"
189+
semver,https://github.com/dtolnay/semver,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
185190
serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar <erick.tryzelaar@gmail.com>, David Tolnay <dtolnay@gmail.com>"
186191
serde-aux,https://github.com/iddm/serde-aux,MIT,Victor Polevoy <maintainer@vpolevoy.com>
187192
serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx
@@ -253,11 +258,15 @@ value-bag,https://github.com/sval-rs/value-bag,Apache-2.0 OR MIT,Ashley Mannix <
253258
want,https://github.com/seanmonstar/want,MIT,Sean McArthur <sean@seanmonstar.com>
254259
wasi,https://github.com/bytecodealliance/wasi,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers
255260
wasip2,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasip2 Authors
261+
wasip3,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasip3 Authors
256262
wasm-bindgen,https://github.com/wasm-bindgen/wasm-bindgen,MIT OR Apache-2.0,The wasm-bindgen Developers
257263
wasm-bindgen-futures,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/futures,MIT OR Apache-2.0,The wasm-bindgen Developers
258264
wasm-bindgen-macro,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/macro,MIT OR Apache-2.0,The wasm-bindgen Developers
259265
wasm-bindgen-macro-support,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/macro-support,MIT OR Apache-2.0,The wasm-bindgen Developers
260266
wasm-bindgen-shared,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/shared,MIT OR Apache-2.0,The wasm-bindgen Developers
267+
wasm-encoder,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wasm-encoder,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Nick Fitzgerald <fitzgen@gmail.com>
268+
wasm-metadata,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wasm-metadata,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasm-metadata Authors
269+
wasmparser,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wasmparser,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Yury Delendik <ydelendik@mozilla.com>
261270
web-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers
262271
web-time,https://github.com/daxpedda/web-time,MIT OR Apache-2.0,The web-time Authors
263272
webpki-roots,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-roots Authors
@@ -283,6 +292,11 @@ windows_x86_64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0
283292
windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft
284293
windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_x86_64_msvc Authors
285294
wit-bindgen,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton <alex@alexcrichton.com>
295+
wit-bindgen-core,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton <alex@alexcrichton.com>
296+
wit-bindgen-rust,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton <alex@alexcrichton.com>
297+
wit-bindgen-rust-macro,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton <alex@alexcrichton.com>
298+
wit-component,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wit-component,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Peter Huene <peter@huene.dev>
299+
wit-parser,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wit-parser,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton <alex@alexcrichton.com>
286300
writeable,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers
287301
yansi,https://github.com/SergioBenitez/yansi,MIT OR Apache-2.0,Sergio Benitez <sb@sergio.bz>
288302
yoke,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar <manishsmail@gmail.com>

‎bottlecap/src/bin/bottlecap/main.rs‎

Lines changed: 14 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -450,10 +450,8 @@ async fn extension_loop_active(
450450
// Wait for any pending flushes
451451
flushing_service.await_handles().await;
452452
// Final flush to capture any data that accumulated since the last
453-
// spawn_non_blocking(). We pass force_stats=true since this is our
454-
// last opportunity to send data before shutdown.
455-
let mut locked_metrics = flushing_service.metrics_flushers().lock().await;
456-
flushing_service.flush_blocking(true, &mut locked_metrics).await;
453+
// spawn_non_blocking(). This is our last opportunity to send data.
454+
flushing_service.flush_blocking_final().await;
457455
break;
458456
}
459457
}
@@ -635,19 +633,13 @@ async fn extension_loop_active(
635633
}
636634
}
637635
_ = race_flush_interval.tick() => {
638-
let mut locked_metrics = metrics_flushers.lock().await;
639-
flushing_service
640-
.flush_blocking(false, &mut locked_metrics)
641-
.await;
636+
flushing_service.flush_blocking().await;
642637
race_flush_interval.reset();
643638
}
644639
}
645640
}
646641
// flush
647-
let mut locked_metrics = metrics_flushers.lock().await;
648-
flushing_service
649-
.flush_blocking(false, &mut locked_metrics)
650-
.await;
642+
flushing_service.flush_blocking().await;
651643
race_flush_interval.reset();
652644
let next_response =
653645
extension::next_event(client, &aws_config.runtime_api, &r.extension_id).await;
@@ -664,10 +656,7 @@ async fn extension_loop_active(
664656
}
665657
}
666658
FlushDecision::Periodic => {
667-
let mut locked_metrics = metrics_flushers.lock().await;
668-
flushing_service
669-
.flush_blocking(false, &mut locked_metrics)
670-
.await;
659+
flushing_service.flush_blocking().await;
671660
race_flush_interval.reset();
672661
}
673662
_ => {
@@ -695,10 +684,7 @@ async fn extension_loop_active(
695684
}
696685
_ = race_flush_interval.tick() => {
697686
if flush_control.flush_strategy == FlushStrategy::Default {
698-
let mut locked_metrics = metrics_flushers.lock().await;
699-
flushing_service
700-
.flush_blocking(false, &mut locked_metrics)
701-
.await;
687+
flushing_service.flush_blocking().await;
702688
race_flush_interval.reset();
703689
}
704690
}
@@ -744,11 +730,8 @@ async fn extension_loop_active(
744730
&lifecycle_listener_shutdown_token,
745731
);
746732

747-
// Final flush with force_stats=true since this is our last opportunity
748-
let mut locked_metrics = metrics_flushers.lock().await;
749-
flushing_service
750-
.flush_blocking(true, &mut locked_metrics)
751-
.await;
733+
// Final flush - this is our last opportunity to send data before shutdown
734+
flushing_service.flush_blocking_final().await;
752735

753736
// Even though we're shutting down, we need to reset the flush interval to prevent any future flushes
754737
race_flush_interval.reset();
@@ -1178,7 +1161,7 @@ async fn start_dogstatsd(
11781161
api_key_factory: Arc<ApiKeyFactory>,
11791162
config: &Arc<Config>,
11801163
) -> (
1181-
Arc<TokioMutex<Vec<MetricsFlusher>>>,
1164+
Arc<Vec<MetricsFlusher>>,
11821165
MetricsAggregatorHandle,
11831166
CancellationToken,
11841167
) {
@@ -1200,17 +1183,20 @@ async fn start_dogstatsd(
12001183
});
12011184

12021185
// Get flushers with aggregator handle
1203-
let flushers = Arc::new(TokioMutex::new(start_metrics_flushers(
1186+
let flushers = Arc::new(start_metrics_flushers(
12041187
Arc::clone(&api_key_factory),
12051188
&aggregator_handle,
12061189
config,
1207-
)));
1190+
));
12081191

12091192
// Create Dogstatsd server
12101193
let dogstatsd_config = DogStatsDConfig {
12111194
host: EXTENSION_HOST.to_string(),
12121195
port: DOGSTATSD_PORT,
12131196
metric_namespace: config.statsd_metric_namespace.clone(),
1197+
so_rcvbuf: config.dogstatsd_so_rcvbuf,
1198+
buffer_size: config.dogstatsd_buffer_size,
1199+
queue_size: config.dogstatsd_queue_size,
12141200
};
12151201
let cancel_token = tokio_util::sync::CancellationToken::new();
12161202
let dogstatsd_agent = DogStatsD::new(

‎bottlecap/src/config/env.rs‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,24 @@ pub struct EnvConfig {
277277
#[serde(deserialize_with = "deserialize_optional_string")]
278278
pub statsd_metric_namespace: Option<String>,
279279

280+
/// @env `DD_DOGSTATSD_SO_RCVBUF`
281+
/// Size of the receive buffer for `DogStatsD` UDP packets, in bytes (`SO_RCVBUF`).
282+
/// Increase to reduce packet loss under high-throughput metric bursts.
283+
#[serde(deserialize_with = "deserialize_option_lossless")]
284+
pub dogstatsd_so_rcvbuf: Option<usize>,
285+
286+
/// @env `DD_DOGSTATSD_BUFFER_SIZE`
287+
/// Maximum size of a single read from any transport (UDP or named pipe), in bytes.
288+
/// Defaults to 8192.
289+
#[serde(deserialize_with = "deserialize_option_lossless")]
290+
pub dogstatsd_buffer_size: Option<usize>,
291+
292+
/// @env `DD_DOGSTATSD_QUEUE_SIZE`
293+
/// Internal queue capacity between the socket reader and metric processor.
294+
/// Defaults to 1024. Increase if the processor can't keep up with burst traffic.
295+
#[serde(deserialize_with = "deserialize_option_lossless")]
296+
pub dogstatsd_queue_size: Option<usize>,
297+
280298
// OTLP
281299
//
282300
// - APM / Traces
@@ -554,6 +572,11 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
554572
config.statsd_metric_namespace = parse_metric_namespace(namespace);
555573
}
556574

575+
// DogStatsD
576+
merge_option!(config, env_config, dogstatsd_so_rcvbuf);
577+
merge_option!(config, env_config, dogstatsd_buffer_size);
578+
merge_option!(config, env_config, dogstatsd_queue_size);
579+
557580
// OTLP
558581
merge_option_to_value!(config, env_config, otlp_config_traces_enabled);
559582
merge_option_to_value!(
@@ -830,6 +853,11 @@ mod tests {
830853
);
831854
jail.set_env("DD_OTLP_CONFIG_LOGS_ENABLED", "true");
832855

856+
// DogStatsD
857+
jail.set_env("DD_DOGSTATSD_SO_RCVBUF", "1048576");
858+
jail.set_env("DD_DOGSTATSD_BUFFER_SIZE", "65507");
859+
jail.set_env("DD_DOGSTATSD_QUEUE_SIZE", "2048");
860+
833861
// AWS Lambda
834862
jail.set_env(
835863
"DD_API_KEY_SECRET_ARN",
@@ -984,6 +1012,9 @@ mod tests {
9841012
otlp_config_traces_probabilistic_sampler_sampling_percentage: Some(50),
9851013
otlp_config_logs_enabled: true,
9861014
statsd_metric_namespace: None,
1015+
dogstatsd_so_rcvbuf: Some(1048576),
1016+
dogstatsd_buffer_size: Some(65507),
1017+
dogstatsd_queue_size: Some(2048),
9871018
api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key"
9881019
.to_string(),
9891020
kms_api_key: "test-kms-key".to_string(),
@@ -1170,4 +1201,43 @@ mod tests {
11701201
Ok(())
11711202
});
11721203
}
1204+
1205+
#[test]
1206+
fn test_dogstatsd_config_from_env() {
1207+
figment::Jail::expect_with(|jail| {
1208+
jail.clear_env();
1209+
jail.set_env("DD_DOGSTATSD_SO_RCVBUF", "1048576");
1210+
jail.set_env("DD_DOGSTATSD_BUFFER_SIZE", "65507");
1211+
jail.set_env("DD_DOGSTATSD_QUEUE_SIZE", "2048");
1212+
1213+
let mut config = Config::default();
1214+
let env_config_source = EnvConfigSource;
1215+
env_config_source
1216+
.load(&mut config)
1217+
.expect("Failed to load config");
1218+
1219+
assert_eq!(config.dogstatsd_so_rcvbuf, Some(1048576));
1220+
assert_eq!(config.dogstatsd_buffer_size, Some(65507));
1221+
assert_eq!(config.dogstatsd_queue_size, Some(2048));
1222+
Ok(())
1223+
});
1224+
}
1225+
1226+
#[test]
1227+
fn test_dogstatsd_config_defaults_to_none() {
1228+
figment::Jail::expect_with(|jail| {
1229+
jail.clear_env();
1230+
1231+
let mut config = Config::default();
1232+
let env_config_source = EnvConfigSource;
1233+
env_config_source
1234+
.load(&mut config)
1235+
.expect("Failed to load config");
1236+
1237+
assert_eq!(config.dogstatsd_so_rcvbuf, None);
1238+
assert_eq!(config.dogstatsd_buffer_size, None);
1239+
assert_eq!(config.dogstatsd_queue_size, None);
1240+
Ok(())
1241+
});
1242+
}
11731243
}

‎bottlecap/src/config/mod.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,17 @@ pub struct Config {
303303
// Metrics
304304
pub metrics_config_compression_level: i32,
305305
pub statsd_metric_namespace: Option<String>,
306+
/// Size of the receive buffer for `DogStatsD` UDP packets, in bytes (`SO_RCVBUF`).
307+
/// Increase to reduce packet loss under high-throughput metric bursts.
308+
/// If None, uses the OS default.
309+
pub dogstatsd_so_rcvbuf: Option<usize>,
310+
/// Maximum size of a single read from any transport (UDP or named pipe), in bytes.
311+
/// Defaults to 8192. For UDP, the client must batch metrics into packets of
312+
/// this size for the increase to take effect.
313+
pub dogstatsd_buffer_size: Option<usize>,
314+
/// Internal queue capacity between the socket reader and metric processor.
315+
/// Defaults to 1024. Increase if the processor can't keep up with burst traffic.
316+
pub dogstatsd_queue_size: Option<usize>,
306317

307318
// OTLP
308319
//
@@ -421,6 +432,14 @@ impl Default for Config {
421432
metrics_config_compression_level: 3,
422433
statsd_metric_namespace: None,
423434

435+
// DogStatsD
436+
// Defaults to None, which uses the OS default.
437+
dogstatsd_so_rcvbuf: None,
438+
// Defaults to 8192 internally.
439+
dogstatsd_buffer_size: None,
440+
// Defaults to 1024 internally.
441+
dogstatsd_queue_size: None,
442+
424443
// OTLP
425444
otlp_config_traces_enabled: true,
426445
otlp_config_traces_span_name_as_resource_name: false,

0 commit comments

Comments
 (0)