diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 717b037b09..6b2df0bf61 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -149,6 +149,7 @@ jobs: run: | if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then export AWS_LC_FIPS_SYS_NO_ASM=1 + git config --global core.longpaths true fi if [[ -z "$CLIPPY_PACKAGES" ]]; then cargo clippy --workspace --all-targets --all-features -- -D warnings diff --git a/.github/workflows/test-ffi.yml b/.github/workflows/test-ffi.yml index 994472b9ec..c1bf2ccdc9 100644 --- a/.github/workflows/test-ffi.yml +++ b/.github/workflows/test-ffi.yml @@ -119,6 +119,9 @@ jobs: env: RUSTFLAGS: "${{ matrix.flags }}" run: | + if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then + git config --global core.longpaths true + fi cargo run --bin release --release -- --out $LIBDD_OUTPUT_FOLDER - name: 'Publish libdatadog' uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # 4.6.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8673c56be6..a86f0f138f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,6 +125,9 @@ jobs: - name: "Remove nextest CI report" shell: bash run: rm -rf target/nextest/ci/junit.xml + - name: Configure git long paths (Windows) + if: runner.os == 'Windows' + run: git config --global core.longpaths true - name: "[${{ steps.rust-version.outputs.version}}] cargo build" shell: bash run: | diff --git a/Cargo.lock b/Cargo.lock index 7b6e9fbe57..d416e2c8be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -397,10 +397,10 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http", + "http 1.1.0", "http-body", "http-body-util", - "itoa", + "itoa 1.0.11", "matchit", "memchr", "mime", @@ -421,7 +421,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", + "http 1.1.0", "http-body", "http-body-util", "mime", @@ -1226,7 +1226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", - "itoa", + "itoa 1.0.11", "ryu", "serde", ] @@ -1447,7 +1447,7 @@ dependencies = [ "bytes", "constcat", "futures", - "http", + "http 1.1.0", "http-body-util", "libdd-common", "libdd-data-pipeline", @@ -1506,7 +1506,7 @@ dependencies = [ "datadog-live-debugger", "datadog-sidecar-macros", "futures", - "http", + "http 1.1.0", "http-body-util", "httpmock", "libc", @@ -1558,7 +1558,7 @@ dependencies = [ "datadog-ipc", "datadog-live-debugger", "datadog-sidecar", - "http", + "http 1.1.0", "libc", "libdd-common", "libdd-common-ffi", @@ -1636,6 +1636,15 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "derp" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b84cfd9b6fa437e498215e5625e9e3ae3bf9bb54d623028a181c40820db169" +dependencies = [ + "untrusted 0.7.1", +] + [[package]] name = "diff" version = "0.1.13" @@ -2141,7 +2150,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.1.0", "indexmap 2.12.1", "slab", "tokio", @@ -2229,7 +2238,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http", + "http 1.1.0", "httpdate", "mime", "sha1", @@ -2241,7 +2250,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http", + "http 1.1.0", ] [[package]] @@ -2320,6 +2329,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.11", +] + [[package]] name = "http" version = "1.1.0" @@ -2328,7 +2348,7 @@ checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 1.0.11", ] [[package]] @@ -2338,7 +2358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.1.0", ] [[package]] @@ -2349,7 +2369,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", + "http 1.1.0", "http-body", "pin-project-lite", ] @@ -2383,7 +2403,7 @@ dependencies = [ "futures-timer", "futures-util", "headers", - "http", + "http 1.1.0", "http-body-util", "hyper", "hyper-util", @@ -2418,11 +2438,11 @@ dependencies = [ "futures-channel", "futures-util", "h2", - "http", + "http 1.1.0", "http-body", "httparse", "httpdate", - "itoa", + "itoa 1.0.11", "pin-project-lite", "smallvec", "tokio", @@ -2435,7 +2455,7 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", + "http 1.1.0", "hyper", "hyper-util", "rustls", @@ -2470,7 +2490,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", + "http 1.1.0", "http-body", "hyper", "ipnet", @@ -2739,6 +2759,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.11" @@ -2851,7 +2877,7 @@ version = "2.0.0" dependencies = [ "anyhow", "bytes", - "http", + "http 1.1.0", "thiserror 1.0.68", ] @@ -2860,7 +2886,7 @@ name = "libdd-capabilities-impl" version = "2.0.0" dependencies = [ "bytes", - "http", + "http 1.1.0", "http-body-util", "libdd-capabilities", "libdd-common", @@ -2880,7 +2906,7 @@ dependencies = [ "futures-core", "futures-util", "hex", - "http", + "http 1.1.0", "http-body", "http-body-util", "httparse", @@ -2941,7 +2967,7 @@ dependencies = [ "cxx-build", "errno", "goblin", - "http", + "http 1.1.0", "libc", "libdd-common", "libdd-libunwind-sys", @@ -2997,7 +3023,7 @@ dependencies = [ "duplicate 2.0.1", "either", "getrandom 0.2.15", - "http", + "http 1.1.0", "http-body-util", "httpmock", "libdd-capabilities", @@ -3072,7 +3098,7 @@ version = "3.0.0" dependencies = [ "anyhow", "cadence", - "http", + "http 1.1.0", "libdd-common", "serde", "tokio", @@ -3196,7 +3222,7 @@ dependencies = [ "cxx-build", "futures", "hashbrown 0.16.1", - "http", + "http 1.1.0", "http-body-util", "httparse", "indexmap 2.12.1", @@ -3268,17 +3294,21 @@ version = "1.0.0" dependencies = [ "anyhow", "base64 0.22.1", + "chrono", "futures", "futures-util", "hashbrown 0.15.1", - "http", + "http 1.1.0", "http-body-util", "hyper", "hyper-util", + "libdd-capabilities", + "libdd-capabilities-impl", "libdd-common", "libdd-remote-config", "libdd-trace-protobuf", "manual_future", + "prost", "serde", "serde_json", "serde_with", @@ -3290,6 +3320,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", + "tuf", "uuid", ] @@ -3340,7 +3371,7 @@ dependencies = [ "bytes", "futures", "hashbrown 0.15.1", - "http", + "http 1.1.0", "http-body-util", "httpmock", "libc", @@ -3439,7 +3470,7 @@ dependencies = [ "async-trait", "criterion", "hashbrown 0.15.1", - "http", + "http 1.1.0", "httpmock", "libdd-capabilities", "libdd-capabilities-impl", @@ -3474,7 +3505,7 @@ dependencies = [ "futures", "getrandom 0.2.15", "hex", - "http", + "http 1.1.0", "http-body", "http-body-util", "httpmock", @@ -3510,7 +3541,7 @@ version = "0.1.0" dependencies = [ "anyhow", "bytes", - "http", + "http 1.1.0", "httpmock", "libdd-common", "libdd-remote-config", @@ -3786,7 +3817,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 1.1.0", "httparse", "memchr", "mime", @@ -4784,7 +4815,7 @@ dependencies = [ "futures-core", "futures-util", "hickory-resolver", - "http", + "http 1.1.0", "http-body", "http-body-util", "hyper", @@ -4827,7 +4858,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.15", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -4991,7 +5022,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -5244,7 +5275,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "indexmap 2.12.1", - "itoa", + "itoa 1.0.11", "memchr", "ryu", "serde", @@ -5306,7 +5337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ "indexmap 2.12.1", - "itoa", + "itoa 1.0.11", "ryu", "serde", "unsafe-libyaml", @@ -5565,7 +5596,7 @@ version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fe17b8deb33a9441280b4266c2d257e166bafbaea6e66b4b34ca139c91766d9" dependencies = [ - "itoa", + "itoa 1.0.11", "ryu", "sval", ] @@ -5576,7 +5607,7 @@ version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "854addb048a5bafb1f496c98e0ab5b9b581c3843f03ca07c034ae110d3b7c623" dependencies = [ - "itoa", + "itoa 1.0.11", "ryu", "sval", ] @@ -5866,7 +5897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", - "itoa", + "itoa 1.0.11", "num-conv", "powerfmt", "serde", @@ -6055,7 +6086,7 @@ dependencies = [ "base64 0.22.1", "bytes", "h2", - "http", + "http 1.1.0", "http-body", "http-body-util", "hyper", @@ -6126,7 +6157,7 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http", + "http 1.1.0", "http-body", "iri-string", "pin-project-lite", @@ -6239,6 +6270,30 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tuf" +version = "0.3.0-beta10" +source = "git+https://github.com/DataDog/rust-tuf/?rev=42fc5b6ac1fe13a997b0e4d0235f0964c18da3fa#42fc5b6ac1fe13a997b0e4d0235f0964c18da3fa" +dependencies = [ + "chrono", + "data-encoding", + "derp", + "futures-io", + "futures-util", + "http 0.2.12", + "itoa 0.4.8", + "log", + "percent-encoding", + "ring", + "serde", + "serde_derive", + "serde_json", + "tempfile", + "thiserror 1.0.68", + "untrusted 0.7.1", + "url", +] + [[package]] name = "twox-hash" version = "1.6.3" @@ -6297,6 +6352,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -6399,7 +6460,7 @@ checksum = "9170e001f458781e92711d2ad666110f153e4e50bfd5cbd02db6547625714187" dependencies = [ "float-cmp", "halfbrown", - "itoa", + "itoa 1.0.11", "ryu", ] diff --git a/datadog-sidecar-ffi/src/lib.rs b/datadog-sidecar-ffi/src/lib.rs index fa41b401d8..2dfacdc96d 100644 --- a/datadog-sidecar-ffi/src/lib.rs +++ b/datadog-sidecar-ffi/src/lib.rs @@ -261,6 +261,7 @@ pub unsafe extern "C" fn ddog_remote_config_reader_for_endpoint<'a>( language: language.to_utf8_lossy().into(), tracer_version: tracer_version.to_utf8_lossy().into(), endpoint: endpoint.clone(), + agentless: None, }, &Arc::new(Target::new( service_name.to_utf8_lossy().to_string(), diff --git a/datadog-sidecar/src/service/sidecar_server.rs b/datadog-sidecar/src/service/sidecar_server.rs index 4565781e33..75c8e67977 100644 --- a/datadog-sidecar/src/service/sidecar_server.rs +++ b/datadog-sidecar/src/service/sidecar_server.rs @@ -800,6 +800,7 @@ impl SidecarInterface for ConnectionSidecarHandler { language: config.language, tracer_version: config.tracer_version, endpoint: config.endpoint, + agentless: None, }, products: config.remote_config_products, capabilities: config.remote_config_capabilities, diff --git a/libdd-common/src/lib.rs b/libdd-common/src/lib.rs index 853ef18d80..c18c65b111 100644 --- a/libdd-common/src/lib.rs +++ b/libdd-common/src/lib.rs @@ -11,7 +11,7 @@ extern crate alloc; use alloc::borrow::Cow; use anyhow::Context; use core::{ops::Deref, str::FromStr}; -use http::uri; +use http::uri::{self, PathAndQuery, Uri}; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::path::PathBuf; @@ -337,6 +337,24 @@ impl Endpoint { /// Default value for the timeout field in milliseconds. pub const DEFAULT_TIMEOUT: u64 = 3_000; + pub fn agentless(site: &str, api_key: String) -> anyhow::Result { + Ok(Self { + url: Uri::builder() + .scheme("https") + .authority( + uri::Authority::try_from(site) + .with_context(|| format!("dd_site is an invalid url: {site}"))?, + ) + .path_and_query(PathAndQuery::from_static("")) + .build() + .with_context(|| format!("rc url is invalid for site: {site}"))?, + api_key: Some(api_key.into()), + timeout_ms: Self::DEFAULT_TIMEOUT, + test_token: None, + use_system_resolver: true, + }) + } + /// Returns an iterator of optional endpoint-specific headers (api-key, test-token) /// as (header_name, header_value) string tuples for any that are available. pub fn get_optional_headers(&self) -> impl Iterator { diff --git a/libdd-remote-config/Cargo.toml b/libdd-remote-config/Cargo.toml index a8744716af..0f27cdbf64 100644 --- a/libdd-remote-config/Cargo.toml +++ b/libdd-remote-config/Cargo.toml @@ -10,7 +10,8 @@ repository = "https://github.com/DataDog/libdatadog/tree/main/libdd-remote-confi description = "Datadog Remote Configuration client and config parsers" [features] -default = ["client", "https"] +default = ["client", "https", "agentless"] +agentless = ["tuf", "chrono", "client", "https"] client = [ "libdd-trace-protobuf", "http-body-util", @@ -23,7 +24,7 @@ client = [ "tokio-util", "manual_future", "time", - "tracing" + "tracing", ] regex-lite = ["libdd-common/regex-lite"] @@ -36,24 +37,35 @@ test = ["hyper/server", "hyper-util"] [dependencies] anyhow = { version = "1.0" } libdd-common = { path = "../libdd-common", version = "5.0.0", default-features = false } +libdd-capabilities = { path = "../libdd-capabilities", version = "2.0.0" } +libdd-capabilities-impl = { version = "2.0.0", path = "../libdd-capabilities-impl", default-features = false } libdd-trace-protobuf = { path = "../libdd-trace-protobuf", version = "3.0.2", optional = true } hyper = { workspace = true, optional = true, default-features = false } -http-body-util = {version = "0.1", optional = true } +http-body-util = { version = "0.1", optional = true } http = { version = "1.1", optional = true } base64 = { version = "0.22.1", optional = true } sha2 = { version = "0.10", optional = true } uuid = { version = "1.7.0", features = ["v4"], optional = true } futures-util = { version = "0.3", optional = true } tokio = { version = "1.36.0", optional = true } -tokio-util = { version = "0.7.10", optional = true } +tokio-util = { version = "0.7.10", optional = true } manual_future = { version = "0.1.1", optional = true } -time = { version = "0.3", features = ["parsing", "serde", "formatting"], optional = true } +time = { version = "0.3", features = [ + "parsing", + "serde", + "formatting", +], optional = true } tracing = { version = "0.1", default-features = false, optional = true } serde = "1.0" serde_json = { version = "1.0", features = ["raw_value"] } serde_with = "3" thiserror = "2" hashbrown = "0.15" +# branch = "opw-develop" +tuf = { git = "https://github.com/DataDog/rust-tuf/", rev = "42fc5b6ac1fe13a997b0e4d0235f0964c18da3fa", default-features = false, optional = true } +chrono = { version = "0.4", default-features = false, features = ["clock"], optional = true } +prost = "0.14.1" +futures = { version = "0.3", features = ["executor"] } # `EnumIter` for external consumers to make struct available to runtime strum = { version = "0.26", default-features = false } strum_macros = "0.26" diff --git a/libdd-remote-config/examples/remote_config_agentless_bench.rs b/libdd-remote-config/examples/remote_config_agentless_bench.rs new file mode 100644 index 0000000000..7a5d00d98e --- /dev/null +++ b/libdd-remote-config/examples/remote_config_agentless_bench.rs @@ -0,0 +1,298 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Benchmark for agentless Remote Config fetching. +//! +//! Measures, for three distinct phases, three quantities each: +//! +//! 1. Client init: `SingleChangesFetcher::new`, which performs the TUF root bootstrap when +//! running in agentless mode. +//! 2. Initial fetch: the first call to `fetch_changes` on a freshly built client. +//! 3. Refetch: the second call to `fetch_changes`, with the client already warm. +//! +//! For each phase we report: +//! +//! * Wall-clock time: total elapsed time, end-to-end. +//! * Poll/CPU time: sum of time spent inside `Future::poll` calls on the current thread. This +//! is an approximation of the active computation time (parsing, TUF verification, request +//! building, response decoding, ...). +//! * Await/IO time: `wall - poll`, the time the future spent suspended waiting for IO (DNS, +//! TCP, TLS handshake, server response, ...). +//! +//! The instrumentation works by polling the benchmarked future manually on a `current_thread` +//! tokio runtime and accumulating the duration of each `poll()` invocation. No additional +//! dependencies are required. +//! +//! Usage: +//! DD_API_KEY=... DD_SITE=datadoghq.com \ +//! cargo run --release --example remote_config_agentless_bench \ +//! -p libdd-remote-config --features agentless +//! +//! Without `DD_API_KEY` / `DD_SITE`, this example exits — agentless mode is required. + +use libdd_common::Endpoint; +use libdd_remote_config::fetch::{ConfigInvariants, ConfigOptions, SingleChangesFetcher}; +use libdd_remote_config::file_storage::ParsedFileStorage; +use libdd_remote_config::RemoteConfigProduct::ApmTracing; +use libdd_remote_config::Target; +use std::future::Future; +use std::pin::Pin; +use std::process::Command; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; + +#[cfg(feature = "agentless")] +use libdd_remote_config::fetch::AgentlessConfig; + +const RUNTIME_ID: &str = "23e76587-5ae1-410c-a05c-137cae600a10"; +const SERVICE: &str = "bench-service"; +const ENV: &str = "bench-env"; +const VERSION: &str = "1.2.3"; + +fn get_hostname() -> String { + Command::new("hostname") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .unwrap_or_else(|| "unknown".to_string()) +} + +/// A future wrapper that accumulates the time spent inside each `poll()` call into +/// `*poll_time`. The wall time is the elapsed between calling `Instrumented::new` (or just +/// before `.await`) and the future completing. +struct Instrumented<'a, F> { + inner: F, + poll_time: &'a mut Duration, +} + +impl<'a, F: Future + Unpin> Instrumented<'a, F> { + fn new(inner: F, poll_time: &'a mut Duration) -> Self { + *poll_time = Duration::ZERO; + Self { inner, poll_time } + } +} + +impl Future for Instrumented<'_, F> { + type Output = F::Output; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let start = Instant::now(); + let res = Pin::new(&mut self.inner).poll(cx); + let elapsed = start.elapsed(); + *self.poll_time += elapsed; + res + } +} + +#[derive(Default, Clone, Copy)] +struct Sample { + wall: Duration, + poll: Duration, +} + +impl Sample { + fn io(&self) -> Duration { + self.wall.saturating_sub(self.poll) + } +} + +fn print_row(label: &str, s: Sample) { + let wall_ms = s.wall.as_secs_f64() * 1000.0; + let poll_ms = s.poll.as_secs_f64() * 1000.0; + let io_ms = s.io().as_secs_f64() * 1000.0; + let poll_pct = if s.wall.as_nanos() > 0 { + 100.0 * s.poll.as_secs_f64() / s.wall.as_secs_f64() + } else { + 0.0 + }; + println!( + " {label:<18} wall = {wall_ms:>9.3} ms poll/CPU = {poll_ms:>9.3} ms \ + await/IO = {io_ms:>9.3} ms ({poll_pct:>5.1}% poll)" + ); +} + +async fn run_one_iteration( + iter: usize, + endpoint: Endpoint, + #[cfg(feature = "agentless")] agentless: Option, + #[cfg(not(feature = "agentless"))] agentless: Option, +) -> anyhow::Result<(Sample, Sample, Sample)> { + let target = Target::new( + SERVICE.to_string(), + ENV.to_string(), + VERSION.to_string(), + vec!["bench:true".to_string()], + vec![], + ); + + let options = ConfigOptions { + invariants: ConfigInvariants { + language: "benchlang".to_string(), + tracer_version: "0.0.1".to_string(), + endpoint, + agentless, + }, + products: vec![ApmTracing], + capabilities: vec![], + }; + + // --- 1. Client init: TUF root bootstrap in agentless mode --- + let mut init_poll = Duration::ZERO; + let init_wall_start = Instant::now(); + let fetcher_fut = Box::pin(SingleChangesFetcher::new( + ParsedFileStorage::default(), + target, + RUNTIME_ID.to_string(), + options, + )); + let mut fetcher = Instrumented::new(fetcher_fut, &mut init_poll).await?; + let init = Sample { + wall: init_wall_start.elapsed(), + poll: init_poll, + }; + + // --- 2. Initial fetch: first call to fetch_changes on a fresh client --- + let mut first_poll = Duration::ZERO; + let first_wall_start = Instant::now(); + // R is inferred from `ParsedFileStorage`'s `UpdatedFiles` impl. + let first_fut = Box::pin(fetcher.fetch_changes()); + let first_changes: Vec<_> = Instrumented::new(first_fut, &mut first_poll).await?; + let first = Sample { + wall: first_wall_start.elapsed(), + poll: first_poll, + }; + + // --- 3. Refetch: second call to fetch_changes (warm client) --- + let mut refetch_poll = Duration::ZERO; + let refetch_wall_start = Instant::now(); + let refetch_fut = Box::pin(fetcher.fetch_changes()); + let refetch_changes: Vec<_> = Instrumented::new(refetch_fut, &mut refetch_poll).await?; + let refetch = Sample { + wall: refetch_wall_start.elapsed(), + poll: refetch_poll, + }; + + println!( + "Iteration #{iter}: initial fetch returned {} change(s), refetch returned {} change(s)", + first_changes.len(), + refetch_changes.len(), + ); + + Ok((init, first, refetch)) +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + let hostname = get_hostname(); + println!("Hostname: {hostname}"); + + let dd_api_key = std::env::var("DD_API_KEY").ok(); + let dd_site = std::env::var("DD_SITE").ok(); + let iterations: usize = std::env::var("BENCH_ITERATIONS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(5); + + let (endpoint, agentless): (Endpoint, Option<_>) = match (dd_api_key, dd_site) { + #[cfg(feature = "agentless")] + (Some(api_key), Some(site)) => { + println!("Agentless mode enabled (site: {site})"); + let endpoint = Endpoint::agentless(&site, api_key) + .expect("Failed to build agentless endpoint from DD_SITE"); + ( + endpoint, + Some(AgentlessConfig { + hostname: hostname.clone(), + ..Default::default() + }), + ) + } + #[cfg(not(feature = "agentless"))] + (Some(_), Some(_)) => { + eprintln!( + "This benchmark requires the `agentless` feature. \ + Re-run with: --features agentless" + ); + std::process::exit(1); + } + _ => { + eprintln!( + "DD_API_KEY and DD_SITE are required for the agentless benchmark.\n\ + Example:\n DD_API_KEY=... DD_SITE=datadoghq.com \\\n \ + cargo run --release --example remote_config_agentless_bench \\\n \ + -p libdd-remote-config --features agentless" + ); + std::process::exit(1); + } + }; + + println!("Running {iterations} iteration(s)\n"); + + let mut inits = Vec::with_capacity(iterations); + let mut firsts = Vec::with_capacity(iterations); + let mut refetches = Vec::with_capacity(iterations); + + for i in 0..iterations { + match run_one_iteration(i, endpoint.clone(), agentless.clone()).await { + Ok((init, first, refetch)) => { + print_row(" client init", init); + print_row(" initial fetch", first); + print_row(" refetch", refetch); + println!(); + inits.push(init); + firsts.push(first); + refetches.push(refetch); + } + Err(e) => { + eprintln!("Iteration {i} failed: {e:?}"); + } + } + } + + if inits.is_empty() { + anyhow::bail!("All iterations failed"); + } + + println!( + "=== Summary over {} successful iteration(s) ===", + inits.len() + ); + print_summary("client init", &inits); + print_summary("initial fetch", &firsts); + print_summary("refetch", &refetches); + + Ok(()) +} + +fn print_summary(label: &str, samples: &[Sample]) { + let n = samples.len() as u32; + let sum_wall: Duration = samples.iter().map(|s| s.wall).sum(); + let sum_poll: Duration = samples.iter().map(|s| s.poll).sum(); + let avg = Sample { + wall: sum_wall / n, + poll: sum_poll / n, + }; + + let min_wall = samples.iter().map(|s| s.wall).min().unwrap(); + let max_wall = samples.iter().map(|s| s.wall).max().unwrap(); + let min_poll = samples.iter().map(|s| s.poll).min().unwrap(); + let max_poll = samples.iter().map(|s| s.poll).max().unwrap(); + + println!("{label}:"); + print_row("avg", avg); + print_row( + "min", + Sample { + wall: min_wall, + poll: min_poll, + }, + ); + print_row( + "max", + Sample { + wall: max_wall, + poll: max_poll, + }, + ); +} diff --git a/libdd-remote-config/examples/remote_config_fetch.rs b/libdd-remote-config/examples/remote_config_fetch.rs index 9686643c15..2984ec0a8e 100644 --- a/libdd-remote-config/examples/remote_config_fetch.rs +++ b/libdd-remote-config/examples/remote_config_fetch.rs @@ -7,7 +7,7 @@ use libdd_remote_config::file_change_tracker::{Change, FilePath}; use libdd_remote_config::file_storage::ParsedFileStorage; use libdd_remote_config::RemoteConfigProduct::ApmTracing; use libdd_remote_config::{RemoteConfigParsed, Target}; -use std::time::Duration; +use std::process::Command; use tokio::time::sleep; const RUNTIME_ID: &str = "23e76587-5ae1-410c-a05c-137cae600a10"; @@ -15,8 +15,70 @@ const SERVICE: &str = "testservice"; const ENV: &str = "testenv"; const VERSION: &str = "1.2.3"; +fn get_hostname() -> String { + Command::new("hostname") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .unwrap_or_else(|| "unknown".to_string()) +} + #[tokio::main(flavor = "current_thread")] async fn main() { + let hostname = get_hostname(); + println!("Hostname: {hostname}"); + + let dd_api_key = std::env::var("DD_API_KEY").ok(); + let dd_site = std::env::var("DD_SITE").ok(); + + let (endpoint, agentless) = match (dd_api_key, dd_site) { + (Some(api_key), Some(site)) => { + #[cfg(feature = "agentless")] + { + use libdd_remote_config::fetch::AgentlessConfig; + println!("DD_API_KEY and DD_SITE are set — enabling agentless mode (site: {site})"); + let endpoint = Endpoint::agentless(&site, api_key) + .expect("Failed to build agentless endpoint from DD_SITE"); + ( + endpoint, + Some(AgentlessConfig { + hostname, + ..Default::default() + }), + ) + } + #[cfg(not(feature = "agentless"))] + { + let _ = (api_key, site); + println!("DD_API_KEY and DD_SITE are set but agentless feature not enabled"); + ( + Endpoint { + url: http::Uri::from_static("http://localhost:8126"), + api_key: None, + timeout_ms: 5000, // custom timeout, defaults to 3 seconds + test_token: None, + ..Default::default() + }, + None, + ) + } + } + _ => { + println!("DD_API_KEY / DD_SITE not set — connecting to local agent"); + ( + Endpoint { + url: http::Uri::from_static("http://localhost:8126"), + api_key: None, + timeout_ms: 5000, // custom timeout, defaults to 3 seconds + test_token: None, + ..Default::default() + }, + None, + ) + } + }; + // SingleChangesFetcher is ideal for a single static (runtime_id, service, env, version) tuple // Otherwise a SharedFetcher (or even a MultiTargetFetcher for a potentially high number of // targets) for multiple targets is needed. These can be manually wired together with a @@ -39,18 +101,15 @@ async fn main() { invariants: ConfigInvariants { language: "awesomelang".to_string(), tracer_version: "99.10.5".to_string(), - endpoint: Endpoint { - url: http::Uri::from_static("http://localhost:8126"), - api_key: None, - timeout_ms: 5000, // custom timeout, defaults to 3 seconds - test_token: None, - ..Default::default() - }, + endpoint, + agentless, }, products: vec![ApmTracing], capabilities: vec![], }, - ); + ) + .await + .expect("Failed to create SingleChangesFetcher"); loop { match fetcher.fetch_changes().await { @@ -81,7 +140,7 @@ async fn main() { } } - sleep(Duration::from_secs(1)).await; + sleep(fetcher.get_refresh_interval()).await; } } diff --git a/libdd-remote-config/roots/gov/config_root.json b/libdd-remote-config/roots/gov/config_root.json new file mode 100644 index 0000000000..40402b5e70 --- /dev/null +++ b/libdd-remote-config/roots/gov/config_root.json @@ -0,0 +1,73 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 1, + "expires": "0001-01-01T00:00:00Z", + "keys": { + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993": { + "keytype": "ecdsa", + "scheme": "ecdsa-sha2-nistp256", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAES2cvfa8r3HZ4AQeDUurdth7xqFk3\nqOuYtR877knUfOtJe+xU/F/ESVrl4B0ZMcyF3TaucgMsae4OVlc2lAW3Nw==\n-----END PUBLIC KEY-----\n" + } + }, + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425": { + "keytype": "ecdsa", + "scheme": "ecdsa-sha2-nistp256", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAETSYtYSgft/owrcf+DsvGzdl+wpSr\nAVe0hmZL/hvdC0oogI98nYTYzycP0B5M5xBeP4ZfJm/mlFFsqCHHosQWkA==\n-----END PUBLIC KEY-----\n" + } + } + }, + "roles": { + "root": { + "keyids": [ + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425", + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425", + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425", + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425", + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425", + "sig": "3046022100f693d8c4ec048f6ac08ab01f5a7cc641c92d3ccf9787b949897da91e57b5c7240221009841eb205814b96b31447bc2893234f691c9ef16d54676f24b94138954cb4d23" + }, + { + "keyid": "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993", + "sig": "3044022060b37818fb24ddca63a7bde900572c833cbf4b157c856734ead79c2f5c28775902207a423e9207db7767aff5863817a089f330af20aa9d21cdeb7f0cc9de337a404f" + } + ] +} \ No newline at end of file diff --git a/libdd-remote-config/roots/gov/director_root.json b/libdd-remote-config/roots/gov/director_root.json new file mode 100644 index 0000000000..058ff2a3b4 --- /dev/null +++ b/libdd-remote-config/roots/gov/director_root.json @@ -0,0 +1,73 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 1, + "expires": "0001-01-01T00:00:00Z", + "keys": { + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10": { + "keytype": "ecdsa", + "scheme": "ecdsa-sha2-nistp256", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEUDdWZozMy6DojzrxkhevLhLzom0E\nnW0C7JWPXgnoL58OHhqDTHhkiUP5H3+fGdVKZ33Vca686aWWSwZUY6xSRQ==\n-----END PUBLIC KEY-----\n" + } + }, + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835": { + "keytype": "ecdsa", + "scheme": "ecdsa-sha2-nistp256", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEE/Evc+4Qx1yPwe0SyvP52C9z8inY\ncTH0eCXHRu+mzShDx7Ne8gyA/vU696i9jcc4pfsOwo1WpIkJsXuqP0jG6A==\n-----END PUBLIC KEY-----\n" + } + } + }, + "roles": { + "root": { + "keyids": [ + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "sig": "3045022016088517105a41506c4465e636483b2eb782d03322da78273a28dd427f2acc78022100cfd965ae1e32c86761d6256a67708e5808f3f98413976373febe23bfefec8da1" + }, + { + "keyid": "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835", + "sig": "3045022007f880ebf48676740f6cb9d216c91e0f51427ef9f44a73f4a7de8da6e4b4b53d022100b0f87441a0761422d7ac57582215d9570de78a0c30dd86479accedf7a00a6a1b" + } + ] +} \ No newline at end of file diff --git a/libdd-remote-config/roots/prod/config_root.json b/libdd-remote-config/roots/prod/config_root.json new file mode 100644 index 0000000000..d5a5eb7894 --- /dev/null +++ b/libdd-remote-config/roots/prod/config_root.json @@ -0,0 +1,63 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 16, + "expires": "2026-10-31T17:00:00Z", + "keys": { + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { "public": "91d413c791907aae0be739d94a1e5e59c5d5ba65a8bbc1fb2153a5680f2d5958" } + }, + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { "public": "9323800f89d833ee263d3661c2616da89e405b92beeec334f21d54b5f60fbd85" } + } + }, + "roles": { + "root": { + "keyids": [ + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "sig": "a8b4ee59576c82bc1bc944df014bbeb90f5cba4ffbd8b7878461da2c934fd3bf93ac4c3b85a7936584da4a5a0cfe93b7150b559fc96423a98a70a11fc844f208" + }, + { + "keyid": "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db", + "sig": "3332e240a023dc267e87e210c7b46b9fa5772932d84936e3a7a5b5018b0f45fbf068ce60b97beb6e7e6c0c12a68d68a44461e590a934b577c71d4ff6dd94db09" + } + ] +} diff --git a/libdd-remote-config/roots/prod/director_root.json b/libdd-remote-config/roots/prod/director_root.json new file mode 100644 index 0000000000..f3882d62a0 --- /dev/null +++ b/libdd-remote-config/roots/prod/director_root.json @@ -0,0 +1,63 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 15, + "expires": "2026-10-31T17:00:00Z", + "keys": { + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { "public": "286d6ae328365afec0f92519ceab68cd627e34072cde90b2f5d167badea970f2" } + }, + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { "public": "afdd68be53815d67f8fa99cf101aac4589a358c660adf7dd4e179fe96834d3c9" } + } + }, + "roles": { + "root": { + "keyids": [ + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8", + "sig": "ccbe8cdd7dfb9a9d6b4bef8075a7aaf9baafe69a07100f22c04677a9737a23b24055ac3a0776c7021ae6a2fd175a251c0604164ea6705a0a896844766d2ecd07" + }, + { + "keyid": "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "sig": "068a2e37e93688702e75ebb328b74cd8879832a63179ba1c54976aae4ee03a5e936c7b7274d4a6aa6755c27cfe800097984d94c83be901bde72103dccebcc008" + } + ] +} diff --git a/libdd-remote-config/roots/staging/config_root.json b/libdd-remote-config/roots/staging/config_root.json new file mode 100644 index 0000000000..9d44633729 --- /dev/null +++ b/libdd-remote-config/roots/staging/config_root.json @@ -0,0 +1,73 @@ +{ + "signatures": [ + { + "keyid": "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545", + "sig": "928d0b9de72a1a1c2fad453e52950509a434814ca0dc5fb43db5100fdbd732461b38b522051ffedc7c226426ce102c245bc69895fde0f0ca0d9615f84027c60f" + }, + { + "keyid": "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "sig": "146d301f5dd97125ddd34d13ad5c7b1f071bbd249d7c86d17a095c0fbfd680ed21737f45997361e14e79be973914cfb35da39c02ce58f81df12afd9eb49d0003" + } + ], + "signed": { + "_type": "root", + "consistent_snapshot": true, + "expires": "2025-12-01T17:00:00Z", + "keys": { + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e": { + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keytype": "ed25519", + "keyval": { + "public": "09402247ef6252018e52c7ba6a3a484936f14dad6ae921c556a1d092f4a68f0f" + }, + "scheme": "ed25519" + }, + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545": { + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keytype": "ed25519", + "keyval": { + "public": "cf248bc222a5dfc9676a2a3ef90526c84adb09649db56686705f69f42908d7d8" + }, + "scheme": "ed25519" + } + }, + "roles": { + "root": { + "keyids": [ + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545" + ], + "threshold": 2 + } + }, + "spec_version": "1.0", + "version": 29 + } +} \ No newline at end of file diff --git a/libdd-remote-config/roots/staging/director_root.json b/libdd-remote-config/roots/staging/director_root.json new file mode 100644 index 0000000000..7361c7b9e9 --- /dev/null +++ b/libdd-remote-config/roots/staging/director_root.json @@ -0,0 +1,73 @@ +{ + "signatures": [ + { + "keyid": "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6", + "sig": "6d7ddf4bcbd1ce223b5352cae4671ef42800d79f0c94dda905cf0dd8a6198ba69795a19201dc7230e4bd872cf109e827233678bf76389910933472417488320e" + }, + { + "keyid": "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1", + "sig": "a1236d12903e1c4024fc6340c50a0f2fe9972e967eb2bace8d6594e156f0466f772bfc0c9f30e07067904073c0d7ba7d48ad00341405312daf0d7bc502ccc50f" + } + ], + "signed": { + "_type": "root", + "consistent_snapshot": true, + "expires": "1970-01-01T00:00:00Z", + "keys": { + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6": { + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keytype": "ed25519", + "keyval": { + "public": "f7c278f32e69ce7d5ca5b81bd2cbe2b4b44177eee36ed025ec06bd19e47eaefe" + }, + "scheme": "ed25519" + }, + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1": { + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keytype": "ed25519", + "keyval": { + "public": "47be15ec10499208aa5ef9a1e32010cc05c047a98d18ad084d6e4e51baa1b93c" + }, + "scheme": "ed25519" + } + }, + "roles": { + "root": { + "keyids": [ + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1", + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1", + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1", + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1", + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6" + ], + "threshold": 2 + } + }, + "spec_version": "1.0", + "version": 1 + } +} \ No newline at end of file diff --git a/libdd-remote-config/src/fetch/agentless.rs b/libdd-remote-config/src/fetch/agentless.rs new file mode 100644 index 0000000000..cf1c5421a3 --- /dev/null +++ b/libdd-remote-config/src/fetch/agentless.rs @@ -0,0 +1,1536 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::fetch::FileStorage; + +use std::{ + borrow::Cow, + fmt, + ops::RangeInclusive, + path::PathBuf, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{bail, format_err}; +use base64::Engine; +use futures::AsyncReadExt as _; +use hashbrown::{HashMap, HashSet}; +use http::{ + header, + uri::{Authority, PathAndQuery}, + Method, Request, Uri, +}; +use libdd_capabilities::{Bytes, HttpClientCapability}; +use libdd_common::Endpoint; +use libdd_trace_protobuf::remoteconfig; +use prost::Message; +use serde_json::Value; +use tracing::{debug, error, warn}; +use tuf::repository::RepositoryStorage; +use tuf::{ + metadata::{ + Metadata, MetadataPath, MetadataVersion, RawSignedMetadata, TargetDescription, TargetPath, + }, + repository::RepositoryProvider as _, +}; + +// Embedded TUF trust roots, per site +const PROD_CONFIG_ROOT: &[u8] = include_bytes!("../../roots/prod/config_root.json"); + +const PROD_DIRECTOR_ROOT: &[u8] = include_bytes!("../../roots/prod/director_root.json"); + +const STAGING_CONFIG_ROOT: &[u8] = include_bytes!("../../roots/staging/config_root.json"); + +const STAGING_DIRECTOR_ROOT: &[u8] = include_bytes!("../../roots/staging/director_root.json"); + +const GOV_CONFIG_ROOT: &[u8] = include_bytes!("../../roots/gov/config_root.json"); + +const GOV_DIRECTOR_ROOT: &[u8] = include_bytes!("../../roots/gov/director_root.json"); + +/// Datadog site selection used to pick a default TUF trust-root pair. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Site { + Prod, + Staging, + Gov, +} + +impl Site { + /// Map an endpoint authority/host to a Datadog site. + /// + /// The configured agentless endpoint authority looks like `config.` + /// (see `make_agentless_configs_endpoint`), so we strip a leading + /// `config.` prefix and apply the same rules the agent uses. + fn from_host(host: &str) -> Self { + let site = host.strip_prefix("config.").unwrap_or(host); + if site == "datad0g.com" || site.ends_with(".datad0g.com") { + Site::Staging + } else if site == "ddog-gov.com" || site.ends_with(".ddog-gov.com") { + Site::Gov + } else { + Site::Prod + } + } + + fn embedded_config_root(self) -> &'static [u8] { + match self { + Site::Prod => PROD_CONFIG_ROOT, + Site::Staging => STAGING_CONFIG_ROOT, + Site::Gov => GOV_CONFIG_ROOT, + } + } + + fn embedded_director_root(self) -> &'static [u8] { + match self { + Site::Prod => PROD_DIRECTOR_ROOT, + Site::Staging => STAGING_DIRECTOR_ROOT, + Site::Gov => GOV_DIRECTOR_ROOT, + } + } +} + +/// Extract the `version` integer from a signed TUF root JSON document. Used +/// only when loading an override root from disk; the embedded roots have +/// their versions hardcoded above. +fn parse_root_version(raw: &[u8]) -> anyhow::Result { + let v: Value = serde_json::from_slice(raw)?; + v.get("signed") + .and_then(|s| s.get("version")) + .and_then(Value::as_u64) + .ok_or_else(|| format_err!("missing or invalid signed.version in TUF root")) +} + +/// Read a TUF root override from disk, returning the bytes and their parsed +/// version +fn load_root(override_path: &std::path::Path) -> anyhow::Result<(Vec, u64)> { + let bytes = std::fs::read(override_path) + .map_err(|e| format_err!("failed to read TUF root override at {override_path:?}: {e}"))?; + let version = parse_root_version(&bytes)?; + Ok((bytes, version)) +} + +const FAKE_AGENT_VERSION: &str = "7.78.4"; + +type TUFRepo = tuf::repository::EphemeralRepository; +type TUFClient = tuf::client::Client; + +// Make a remote config API endpoint from and endpoint where `e.url` is the base dd site +// If the endpoint is not suitable (api key not set, not https), returns N +pub fn make_agentless_configs_endpoint(e: &Endpoint) -> Option { + let e = e.clone(); + if !(e.url.scheme_str().is_some_and(|s| s == "https") + && e.url.authority().is_some() + && e.api_key.is_some()) + { + return None; + } + + let mut parts = e.url.into_parts(); + parts.authority = + Some(Authority::try_from(format!("config.{}", parts.authority?.as_str())).ok()?); + parts.path_and_query = Some(PathAndQuery::from_static("/api/v0.1/configurations")); + + Some(Endpoint { + url: Uri::from_parts(parts).ok()?, + ..e + }) +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Default)] +pub struct AgentlessConfig { + /// Hostname reported to the RC backend + /// Must be non empty in agentless mode; an empty value causes + /// `ConfigFetcherState::new` to downgrade to agent mode. + pub hostname: String, + /// Optional path to a TUF repo root JSON to use instead of the + /// embedded one + pub config_root_override_path: Option, + pub director_root_override_path: Option, + /// Override the `agent_uuid` field sent to the RC backend. + pub agent_uuid: Option, +} + +pub type NativeAgentlessFetcher = AgentlessFetcher; + +pub struct AgentlessFetcher { + http: C, + opaque_backend_state: Vec, + director_client: TUFClient, + config_client: TUFClient, + /// Raw signed TUF root bytes used to (re)build the clients. Usually a static + /// slice for embedded roots, owned only when loaded from an override path. + config_root_bytes: Cow<'static, [u8]>, + director_root_bytes: Cow<'static, [u8]>, + /// Last non-empty config top-targets metadata received from the backend. The + /// backend only re-sends config top-targets when their version changes; on a + /// config root rotation rust-tuf purges its trusted top-targets and must + /// re-fetch them from the remote repo, so we cache and re-serve the last copy + /// to avoid being stuck (incident-45734). + last_config_top_targets: Option, + /// Org UUID pinned to a config root version. Root rotation forces a + /// re-fetch, so a bad pin clears itself on the next rotation. + org_uuid: Option, + /// Whether the one-shot org-UUID prefetch has run. Cleared by `reset()`. + org_data_prefetched: bool, + hostname: String, + agent_uuid_override: Option, + products: HashSet, + refresh_interval: Duration, + /// Number of consecutive `fetch_config` failures. Reset to 0 on success. + consecutive_failures: u32, + endpoint: Endpoint, +} + +#[derive(Debug, Clone)] +struct OrgUuidBinding { + config_root_version: u64, + uuid: String, +} + +#[derive(Debug)] +pub struct ClientTargetRef { + pub path: String, + pub version: u64, + pub primary_hash: String, + pub length: u64, +} + +pub struct ClientResponse { + pub root_version: u64, + pub target_version: u64, + pub opaque_backend_state: Vec, + /// All currently active targets; content is already stored in the outer cache. + pub targets: Vec, + pub refresh_interval: Duration, +} + +/// A trusted, unexpired TUF target. Produced by [`trusted_targets`]. +struct TrustedTarget<'a> { + path: &'a tuf::metadata::TargetPath, + length: u64, + version: u64, + all_hashes: Vec<(&'static tuf::crypto::HashAlgorithm, tuf::crypto::HashValue)>, + /// Lowercase hex of the first supported hash; used for cache-hit comparisons. + primary_hash: String, +} + +const CUSTOM_METADATA_EXPIRY_PATH: &str = "expires"; + +impl<'a> TrustedTarget<'a> { + fn try_create(path: &'a TargetPath, desc: &'a TargetDescription) -> anyhow::Result { + if let Some(expiry) = desc.custom().get(CUSTOM_METADATA_EXPIRY_PATH) { + let expiry_ts = expiry + .as_u64() + .ok_or_else(|| format_err!("expiry not a number"))?; + + // Use saturating arithmetic so a far-future `expires` cannot overflow + // `u64` (which panics in debug builds and wraps to a fail-open value in + // release builds). Saturating to `u64::MAX` keeps genuinely far-future + // targets "not yet expired" while never wrapping below `now`. + if expiry_ts.saturating_mul(1000) <= now_unix_milli_ts() { + bail!("expired target at path: {path}") + } + } + + let all_hashes = tuf::crypto::retain_supported_hashes(desc.hashes()); + if all_hashes.is_empty() { + bail!("no supported hash algorithm for target at path: {path}") + } + // retain_supported_hashes return order is deterministic. + let primary_hash = all_hashes[0].1.to_string(); + + let version = desc.custom().get("v").and_then(|v| v.as_u64()).unwrap_or(0); + + Ok(Self { + path, + length: desc.length(), + version, + all_hashes, + primary_hash, + }) + } +} + +impl AgentlessFetcher { + /// Create a new `AgentlessFetcher` client. + /// + /// # Errors + /// Returns an error if TUF root initialization fails. + pub async fn new(cfg: AgentlessConfig, endpoint: Endpoint) -> anyhow::Result { + // Pick the default trust roots based on the endpoint's host and overrides + let site = endpoint + .url + .host() + .map(Site::from_host) + .unwrap_or(Site::Prod); + + let config_root_bytes: Cow<'static, [u8]> = match cfg.config_root_override_path.as_deref() { + Some(p) => Cow::Owned(load_root(p)?.0), + None => Cow::Borrowed(site.embedded_config_root()), + }; + let director_root_bytes: Cow<'static, [u8]> = + match cfg.director_root_override_path.as_deref() { + Some(p) => Cow::Owned(load_root(p)?.0), + None => Cow::Borrowed(site.embedded_director_root()), + }; + + Ok(Self { + endpoint, + http: C::new_client(), + director_client: TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(director_root_bytes.to_vec()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await?, + config_client: TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(config_root_bytes.to_vec()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await?, + config_root_bytes, + director_root_bytes, + last_config_top_targets: None, + org_uuid: None, + org_data_prefetched: false, + hostname: cfg.hostname, + agent_uuid_override: cfg.agent_uuid, + products: HashSet::new(), + + opaque_backend_state: Vec::new(), + refresh_interval: Duration::from_secs(60), + consecutive_failures: 0, + }) + } + + /// Number of consecutive failed `fetch_config` calls. `0` after a success. + pub fn consecutive_failures(&self) -> u32 { + self.consecutive_failures + } + + /// Recommended delay before the next `fetch_config` attempt given the + /// current consecutive-failure count. Returns `None` when no backoff + /// applies (i.e. either no failures yet, or only a single one). + pub fn next_backoff(&self) -> Option { + compute_backoff(self.consecutive_failures) + } + + /// Rebuild both TUF clients from the embedded/override roots and discard all + /// derived state, restarting the fetcher as if freshly constructed. Called on + /// `apply()` failure so a partially-advanced trusted database cannot block + /// subsequent polls + /// + /// TODO(rust-tuf): rebuilding from the embedded root discards any newer root + /// versions we had already verified, so recovery re-reports the embedded root + /// version and the backend re-sends the rotated roots to be re-verified. + /// rust-tuf has a private `Database::purge_metadata()` (TUF §5.1.9) that + /// clears snapshot/targets/timestamp/delegations while keeping the trusted + /// root; if that were exposed we could reset non-root state in place and + /// preserve the advanced root instead of restarting from the embedded one. + async fn reset(&mut self) -> anyhow::Result<()> { + self.director_client = TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(self.director_root_bytes.to_vec()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await?; + self.config_client = TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(self.config_root_bytes.to_vec()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await?; + self.products.clear(); + self.opaque_backend_state.clear(); + self.last_config_top_targets = None; + self.org_uuid = None; + self.org_data_prefetched = false; + Ok(()) + } + + /// Check the config snapshot's `custom.org_uuid` against the UUID served + /// by `/api/v0.1/org`. No snapshot custom => skip. + /// + /// The pinned UUID is keyed by the config trusted-root version, so a root + /// rotation forces a fresh fetch. `prefetched` reuses the first-poll + /// concurrent fetch; otherwise the UUID is fetched here. + async fn verify_org_uuid(&mut self, prefetched: Option) -> anyhow::Result<()> { + let Some(expected) = self + .config_client + .database() + .trusted_snapshot() + .ok_or_else(|| format_err!("org UUID check failed: missing trusted snapshot"))? + .additional_fields() + .get("custom") + .and_then(|c| c.get("org_uuid")) + .and_then(Value::as_str) + else { + return Ok(()); + }; + + let root_version = self.config_client.database().trusted_root().version(); + + let stored: &str = match self.org_uuid.as_ref() { + Some(b) if b.config_root_version == root_version => &b.uuid, + _ => { + let uuid = match prefetched { + Some(u) => u, + None => self.get_org_data().await?.uuid, + }; + &self + .org_uuid + .insert(OrgUuidBinding { + config_root_version: root_version, + uuid, + }) + .uuid + } + }; + + anyhow::ensure!( + stored == expected, + "org UUID mismatch: intake={stored} snapshot={expected}" + ); + Ok(()) + } + + async fn fetch_target(&self, target: &TrustedTarget<'_>) -> anyhow::Result> { + let target_path = target.path; + + // Fetch from the remote __unverified__ repo. + // This is fine because we compare the hash+len against TUF-validated metadata. + let mut read = self + .director_client + .remote_repo() + .fetch_target(target_path) + .await?; + let mut buf = Vec::new(); + read.read_to_end(&mut buf).await?; + + if buf.len() as u64 != target.length { + bail!("bad length for file at path: {}", target.path) + } + + let hash_algs = target + .all_hashes + .iter() + .map(|(alg, _val)| (*alg).clone()) + .collect::>(); + let actual_hashes = tuf::crypto::calculate_hashes_from_slice(&buf, hash_algs.as_slice())?; + let expected: HashMap<_, _> = target + .all_hashes + .iter() + .map(|(alg, val)| (alg, val)) + .collect(); + + if !(actual_hashes.len() == expected.len() + && actual_hashes + .iter() + .all(|(k, v)| expected.get(&k).is_some_and(|e| *e == v))) + { + bail!("hash did not match: {}", target.path) + } + + Ok(buf) + } + + /// Fetch remote config. Newly-downloaded target content is written directly into + /// `cache` rather than buffered inside the agentless fetcher. + pub(crate) async fn fetch_config( + &mut self, + c: remoteconfig::Client, + cache: &TargetCache<'_, Storage>, + ) -> anyhow::Result { + // Derive the versions we report to the backend directly from the live + // trusted databases so they always match what `apply()` has actually + // committed. (Previously these came from a `self.initialized` flag that + // could diverge from the in-place-mutated trusted DB after a partial + // `apply()` failure — D-F1.) A freshly built or just-reset client has no + // trusted snapshot yet, so it reports snapshot version 0 and the embedded + // root versions. + let current_config_snapshot_version = self + .config_client + .database() + .trusted_snapshot() + .map_or(0, |s| s.version()); + let current_config_root_version = self.config_client.database().trusted_root().version(); + let current_director_root_version = + self.director_client.database().trusted_root().version(); + + let all_products = c.products.iter().fold(HashSet::new(), |mut acc, p| { + acc.get_or_insert_with(p, String::clone); + acc + }); + let new_products = all_products + .difference(&self.products) + .cloned() + .collect::>(); + let old_products = self + .products + .intersection(&all_products) + .cloned() + .collect::>(); + + let now = now_unix_milli_ts(); + + let (has_error, error) = match c.state.as_ref() { + Some(state) if state.has_error => (true, state.error.clone()), + _ => (false, String::new()), + }; + + let request = remoteconfig::LatestConfigsRequest { + hostname: self.hostname.clone(), + current_config_snapshot_version, + current_config_root_version, + current_director_root_version, + products: old_products, + new_products, + backend_client_state: self.opaque_backend_state.clone(), + active_clients: vec![remoteconfig::Client { + last_seen: now, + ..c + }], + agent_version: FAKE_AGENT_VERSION.to_owned(), + has_error, + error, + trace_agent_env: String::new(), + org_uuid: String::new(), + tags: vec![], + agent_uuid: self + .agent_uuid_override + .as_deref() + .unwrap_or_else(|| libdd_common::machine_id::get_machine_id()) + .to_owned(), + }; + // During first poll only fetch org data in parallel with the config request + // to hide its latency. Later polls (and prefetch failures) fall back to + // the sequential fetch in `verify_org_uuid`. + let (response_result, prefetched_org_uuid) = if !self.org_data_prefetched { + self.org_data_prefetched = true; + let (r, org) = futures::join!(self.get_latest_config(request), self.get_org_data()); + let org_uuid = match org { + Ok(d) => Some(d.uuid), + Err(e) => { + debug!("org data prefetch failed, will fetch lazily: {e:#}"); + None + } + }; + (r, org_uuid) + } else { + (self.get_latest_config(request).await, None) + }; + let response = match response_result { + Ok(r) => r, + Err(e) => { + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + return Err(e); + } + }; + + let active_targets = match self.apply(&response, cache, prefetched_org_uuid).await { + Ok(t) => t, + Err(e) => { + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + // On any `apply()` failure the trusted databases may have been advanced + // in place and incrementally, leaving them inconsistent with the + // versions we would report next poll. + // Reset both clients so the next poll restarts from a clean state + if let Err(reset_err) = self.reset().await { + error!("failed to reset TUF clients after apply error: {reset_err}"); + } + return Err(e); + } + }; + self.consecutive_failures = 0; + + self.products = all_products; + + // TODO: + // In the future we will want to query configs for multiple clients (for PHP, which can have + // many processes use the same rc client). + // This means we will need to dispatch the different files based on filter predicates + // which we currently do not parse. + + Ok(ClientResponse { + root_version: self.config_client.database().trusted_root().version(), + target_version: self + .config_client + .database() + .trusted_targets() + .ok_or(anyhow::anyhow!("Missing target data"))? + .version(), + opaque_backend_state: self.opaque_backend_state.clone(), + targets: active_targets, + refresh_interval: self.refresh_interval, + }) + } + + /// Query the Remote Config org-status endpoint. + /// + /// # Errors + /// Returns an error if the HTTP request fails or the response cannot be decoded. + pub async fn get_org_status(&self) -> anyhow::Result { + let path = PathAndQuery::from_static("/api/v0.1/status"); + let res = self.send_request(Method::GET, path, Bytes::new()).await?; + parse_rc_response(res) + } + + pub async fn get_org_data(&self) -> anyhow::Result { + let path = PathAndQuery::from_static("/api/v0.1/org"); + let res = self.send_request(Method::GET, path, Bytes::new()).await?; + parse_rc_response(res) + } + + /// Fetch the latest Remote Config for this client. + /// + /// # Errors + /// Returns an error if the HTTP request fails or the response cannot be decoded. + async fn get_latest_config( + &self, + req: remoteconfig::LatestConfigsRequest, + ) -> anyhow::Result { + let path = PathAndQuery::from_static("/api/v0.1/configurations"); + let body = Bytes::from(req.encode_to_vec()); + let res = self.send_request(Method::POST, path, body).await?; + let res = parse_rc_response(res)?; + Ok(res) + } + + #[allow(clippy::future_not_send)] + async fn send_request( + &self, + method: Method, + path: PathAndQuery, + body: Bytes, + ) -> anyhow::Result> { + let req = self + .endpoint + .set_standard_headers( + Request::builder(), + concat!("Libdatadog/", env!("CARGO_PKG_VERSION")), + ) + .header(header::CONTENT_TYPE, "application/x-protobuf") + .uri(url_with_path(self.endpoint.url.clone(), path)?) + .method(method) + .body(body)?; + let timeout = Duration::from_millis(self.endpoint.timeout_ms); + let response = tokio::time::timeout(timeout, self.http.request(req)) + .await + .map_err(|_| { + format_err!( + "Remote config request timed out after {}ms", + self.endpoint.timeout_ms, + ) + })??; + Ok(response) + } + + async fn apply( + &mut self, + response: &remoteconfig::LatestConfigsResponse, + cache: &TargetCache<'_, S>, + prefetched_org_uuid: Option, + ) -> anyhow::Result> { + // At a high level, we're populating the "remote" repos with the metadata + // that we received from upstream (which does not validate it), and then using the clients' + // `update` methods to synchronize that metadata to the "local" repos, during which + // validation is performed. + + let root_path = MetadataPath::root(); + let timestamp_path = MetadataPath::timestamp(); + let snapshot_path = MetadataPath::snapshot(); + let targets_path = MetadataPath::targets(); + + let Some(metas) = response.config_metas.as_ref() else { + bail!("missing config meta from LatestConfigsResponse") + }; + // The backend diffs config top-targets against the snapshot version we + // report: it sends them only when their version changed, otherwise the + // field is empty. + // + // rust-tuf purges its trusted top-targets whenever the + // config root rotates and then re-fetches them from the remote repo; if + // we wipe the remote repo (below) and the backend sent none, that + // re-fetch finds nothing and `update()` is stuck (incident-45734). Re-serve + // the last cached copy when the response omits them. + let config_top_targets = if metas.top_targets.is_some() { + &metas.top_targets + } else { + &self.last_config_top_targets + }; + + let config_repo_mut = self.config_client.remote_repo_mut(); + *config_repo_mut = TUFRepo::new(); + + store(config_repo_mut, &root_path, &metas.roots).await?; + store_noversion(config_repo_mut, ×tamp_path, &metas.timestamp).await?; + store(config_repo_mut, &snapshot_path, &metas.snapshot).await?; + store(config_repo_mut, &targets_path, config_top_targets).await?; + // Delegated targets are stored later, after verifying the top-level signatures. + + let Some(metas) = response.director_metas.as_ref() else { + bail!("missing director meta from LatestConfigsResponse") + }; + + let director_remote_repo = self.director_client.remote_repo_mut(); + *director_remote_repo = TUFRepo::new(); + for target_file in &response.target_files { + let trimmed_path = trim_hash_target_path(&target_file.path)?; + director_remote_repo + .store_target( + &TargetPath::new(&trimmed_path)?, + &mut target_file.raw.as_slice(), + ) + .await?; + } + + store(director_remote_repo, &root_path, &metas.roots).await?; + store_noversion(director_remote_repo, ×tamp_path, &metas.timestamp).await?; + store(director_remote_repo, &snapshot_path, &metas.snapshot).await?; + store(director_remote_repo, &targets_path, &metas.targets).await?; + + self.config_client.update().await?; + self.director_client.update().await?; + + let now = chrono::Utc::now(); + let parent = MetadataPath::targets(); + + // Ingest each delegated targets blob into the config DB. This enforces the + // per-product signing keys: `update_delegated_targets` verifies signatures, + // expiry and version monotonicity before inserting into `trusted_delegations`. + if let Some(metas) = response.config_metas.as_ref() { + for dm in &metas.delegated_targets { + let role = MetadataPath::new(dm.role.clone()) + .map_err(|e| format_err!("bad delegated role name {:?}: {e}", dm.role))?; + let raw = RawSignedMetadata::new(dm.raw.clone()); + self.config_client + .database_mut() + .update_delegated_targets(&now, &parent, &role, &raw) + .map_err(|e: tuf::Error| { + format_err!("failed to verify config delegation {}: {e}", dm.role) + })?; + } + } + + // Enforce that each director-announced target is also authorized by the + // config repo's per-product delegated keys, not just the single director key. + verify_director_against_config(&self.config_client, &self.director_client)?; + self.verify_org_uuid(prefetched_org_uuid).await?; + + let targets: Vec> = trusted_targets(&self.director_client)? + .filter(|t| { + let parseable = cache.is_parseable_path(t.path.as_str()); + if !parseable { + warn!( + "Skipping unparseable/unknown-product remote config path: {}", + t.path.as_str() + ); + } + parseable + }) + .collect(); + + let cached_paths: hashbrown::HashSet<&str> = cache.is_cached_batch( + targets + .iter() + .map(|t| (t.path.as_str(), t.primary_hash.as_str(), t.length)), + ); + + let mut new_targets: Vec = Vec::new(); + for t in &targets { + if cached_paths.contains(t.path.as_str()) { + continue; + } + let content = self.fetch_target(t).await?; + new_targets.push(NewTarget { + path: t.path.as_str().to_owned(), + version: t.version, + primary_hash: t.primary_hash.clone(), + hashes: t + .all_hashes + .iter() + .map(|(alg, hash)| (hash_algorithm_to_str(alg).to_owned(), hash.to_string())) + .collect(), + content, + }); + } + cache.store_batch(new_targets)?; + + let active_path_strs: hashbrown::HashSet<&str> = + targets.iter().map(|t| t.path.as_str()).collect(); + cache.retain_only(&active_path_strs); + + let active_targets: Vec = targets + .iter() + .map(|t| ClientTargetRef { + path: t.path.as_str().to_owned(), + version: t.version, + primary_hash: t.primary_hash.clone(), + length: t.length, + }) + .collect(); + + // The Remote Config service uses a `custom` field at the top-level of the targets + // metadata to store this field which we are supposed to echo back to the server. + if let Some((opaque_backend_state, refresh_interval)) = + get_director_custom(&self.director_client) + { + if let Some(opaque_backend_state) = opaque_backend_state { + self.opaque_backend_state = opaque_backend_state; + } + if let Some(refresh_interval) = refresh_interval { + self.refresh_interval = refresh_interval; + } + } + + // Commit the top-targets cache only now that `apply()` has fully + // succeeded, so a mid-way failure (followed by `reset()`) never leaves a + // stale cached copy + if let Some(config_metas) = response.config_metas.as_ref() { + if config_metas.top_targets.is_some() { + self.last_config_top_targets = config_metas.top_targets.clone(); + } + } + + Ok(active_targets) + } +} + +const REFRESH_INTERVAL_BOUNDS: RangeInclusive = + Duration::from_secs(1)..=Duration::from_secs(60); + +fn get_director_custom(director_client: &TUFClient) -> Option<(Option>, Option)> { + let custom = director_client + .database() + .trusted_targets()? + .additional_fields() + .get("custom")?; + + Some(( + custom + .get("opaque_backend_state") + .and_then(Value::as_str) + .and_then(|s| base64::engine::general_purpose::STANDARD.decode(s).ok()), + custom + .get("agent_refresh_interval") + .and_then(Value::as_u64) + .map(Duration::from_secs) + // Mirror the agent: silently drop values outside `[1s, 1m]` + .filter(|d| REFRESH_INTERVAL_BOUNDS.contains(d)), + )) +} + +fn url_with_path(base: http::Uri, path: PathAndQuery) -> anyhow::Result { + let mut parts = base.into_parts(); + parts.path_and_query = Some(path); + Ok(http::Uri::from_parts(parts)?) +} + +fn parse_rc_response( + response: http::Response, +) -> anyhow::Result { + let status = response.status().as_u16(); + let body = response.into_body(); + if !(200..300).contains(&status) { + bail!( + "Non 2XX status code: {}\n{}", + status, + String::from_utf8_lossy(&body) + ) + } + Ok(T::decode(body)?) +} + +/// Compute the backoff delay to wait before the next `fetch_config` attempt, +/// given the number of consecutive failures observed so far. +fn compute_backoff(consecutive_failures: u32) -> Option { + match consecutive_failures { + 0 | 1 => None, + 2 => Some(jitter_secs(30, 60)), + 3 => Some(jitter_secs(60, 120)), + _ => Some(Duration::from_secs(120)), + } +} + +/// Pseudo-random duration in `[min_secs, max_secs]`, derived from the current +/// wall-clock subsecond nanos. This is sufficient for jitter purposes (we do +/// not need cryptographic randomness here, and pulling in a `rand` dependency +/// for one usage is overkill). +fn jitter_secs(min_secs: u64, max_secs: u64) -> Duration { + let (lo, hi) = if min_secs <= max_secs { + (min_secs, max_secs) + } else { + (max_secs, min_secs) + }; + let span = hi.saturating_sub(lo); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| u64::from(d.subsec_nanos())) + .unwrap_or(0); + let offset = if span == 0 { 0 } else { nanos % (span + 1) }; + Duration::from_secs(lo + offset) +} + +fn now_unix_milli_ts() -> u64 { + u64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_millis(), + ) + .unwrap_or(u64::MAX) +} + +/// Cross-verify the director's announced targets against the config repo's +/// per-product delegation tree. +/// +/// Datadog's RC uses two TUF repos: the *director* has a flat top-level +/// `targets` signed by a single key, while the *config repo* authorizes +/// everything through per-product delegated roles (`APM_TRACING`, `ASM_DD`, …). +/// Trusting only the director collapses the model to one key, so every director +/// target must also be authorized — with the same `(length, hashes)` — by the +/// config delegation tree. +/// +/// We can't use the built-in `Database::target_description` walker because it +/// doesn't filter delegations by `paths` before checking `trusted_delegations`, +/// and its matcher handles only directory-prefix patterns, not the globs +/// Datadog uses (`datadog/*/APM_TRACING/*/*`). +/// +/// Assumes the config repo is flat (delegated roles are direct children of the +/// top-level targets); nested delegations would require making this recursive. +fn verify_director_against_config( + config_client: &TUFClient, + director_client: &TUFClient, +) -> anyhow::Result<()> { + let top_config_targets = config_client + .database() + .trusted_targets() + .ok_or_else(|| format_err!("config client has no trusted top-level targets"))?; + let trusted_delegations = config_client.database().trusted_delegations(); + + let director_targets = director_client + .database() + .trusted_targets() + .ok_or_else(|| format_err!("director client has no trusted targets"))?; + + for (path, dir_desc) in director_targets.targets() { + let cfg_desc = lookup_config_target(path, top_config_targets, trusted_delegations) + .ok_or_else(|| { + format_err!("director target {path} is not authorized by config delegations") + })?; + + if cfg_desc.length() != dir_desc.length() { + bail!( + "length mismatch between director and config for {path}: director={}, config={}", + dir_desc.length(), + cfg_desc.length() + ); + } + + // Check that the director and config hases sets are equal + if dir_desc.hashes() != cfg_desc.hashes() { + bail!("hash set mismatch between director and config for {path}"); + } + } + + Ok(()) +} + +/// Resolve `path` against the (flat) config delegation tree, returning the +/// target description from the first matching delegation that lists it. +/// Returns `None` if no delegation authorizes the path. +fn lookup_config_target<'a>( + path: &TargetPath, + top: &'a tuf::verify::Verified, + trusted_delegations: &'a std::collections::HashMap< + MetadataPath, + tuf::verify::Verified, + >, +) -> Option<&'a TargetDescription> { + // Direct hit on the top-level targets (Datadog's are empty, but be safe). + if let Some(d) = top.targets().get(path) { + return Some(d); + } + + // Spec-style preorder walk over the (ordered) delegation list. + for delegation in top.delegations().roles() { + let matches_scope = delegation + .paths() + .iter() + .any(|pat| target_matches_pattern(path.as_str(), pat.as_str())); + if !matches_scope { + continue; + } + + if let Some(meta) = trusted_delegations.get(delegation.name()) { + if let Some(d) = meta.targets().get(path) { + return Some(d); + } + } + + // Scope matched but path not found: per TUF, a `terminating` delegation + // stops the search rather than falling through to siblings. + if delegation.terminating() { + return None; + } + } + + None +} + +/// TUF-style glob path matching. `*` matches any run of characters within a +/// single `/`-delimited segment; segments must otherwise match literally. +/// E.g. `datadog/*/APM_TRACING/*/*` matches +/// `datadog/556989/APM_TRACING//` but not +/// `datadog/x/y/APM_TRACING//`. +fn target_matches_pattern(path: &str, pattern: &str) -> bool { + let mut p_segs = pattern.split('/'); + let mut t_segs = path.split('/'); + loop { + match (p_segs.next(), t_segs.next()) { + (None, None) => return true, + (Some(p), Some(t)) if segment_matches(p, t) => continue, + _ => return false, + } + } +} + +/// Match a single path segment against a single pattern segment. `*` in the +/// pattern matches zero-or-more characters (within the segment, since segments +/// don't contain `/`). +fn segment_matches(pattern: &str, segment: &str) -> bool { + // Fast paths. + if pattern == "*" { + return true; + } + if !pattern.contains('*') { + return pattern == segment; + } + + let wildcard_count = pattern.as_bytes().iter().filter(|c| **c == b'*').count(); + if wildcard_count == 1 { + let Some((start, end)) = pattern.split_once('*') else { + // unreachable in practice, if wildcard_count == 1 then we always have two parts in the + // pattern + return false; + }; + return segment.starts_with(start) && segment.ends_with(end); + } + + // General case: literals split by `*`, anchored at both ends. + let parts: Vec<&str> = pattern.split('*').collect(); + let first = parts[0]; + if !segment.starts_with(first) { + return false; + } + let last = parts[parts.len() - 1]; + let mut cursor = &segment[first.len()..]; + // Middle literals must appear in order, non-overlapping. + for mid in &parts[1..parts.len() - 1] { + if mid.is_empty() { + continue; + } + match cursor.find(mid) { + Some(i) => cursor = &cursor[i + mid.len()..], + None => return false, + } + } + cursor.ends_with(last) && cursor.len() >= last.len() +} + +/// Return all currently trusted, unexpired targets. Targets that are expired or that lack a +/// supported hash algorithm are skipped with a debug log. +fn trusted_targets( + director_client: &TUFClient, +) -> anyhow::Result> + '_> { + Ok(director_client + .database() + .trusted_targets() + .ok_or_else(|| format_err!("missing targets from TUF director client"))? + .targets() + .iter() + .filter_map(|(path, desc)| { + TrustedTarget::try_create(path, desc) + .inspect_err(|e| { + debug!(%path, "Skipping target: error {}", e); + }) + .ok() + })) +} + +async fn store<'a, T>(repo: &mut TUFRepo, path: &MetadataPath, tms: T) -> anyhow::Result<()> +where + T: IntoIterator + 'a, +{ + for tm in tms { + repo.store_metadata( + path, + MetadataVersion::Number(tm.version), + &mut tm.raw.as_slice(), + ) + .await?; + } + Ok(()) +} + +async fn store_noversion( + repo: &mut TUFRepo, + path: &MetadataPath, + tms: &Option, +) -> anyhow::Result<()> { + if let Some(tm) = tms { + repo.store_metadata(path, MetadataVersion::None, &mut tm.raw.as_slice()) + .await?; + } + Ok(()) +} + +fn hash_algorithm_to_str(alg: &tuf::crypto::HashAlgorithm) -> &str { + match alg { + tuf::crypto::HashAlgorithm::Sha256 => "sha256", + tuf::crypto::HashAlgorithm::Sha512 => "sha512", + tuf::crypto::HashAlgorithm::Unknown(s) => s.as_str(), + _ => "unknown", + } +} + +/// Strip the leading `.` prefix from the basename of a TUF target path. +/// For instance "datadog/2///.config"` => `"datadog/2///config"` +/// +/// See https://datadoghq.atlassian.net/browse/RC-1859 for more information. +fn trim_hash_target_path(target_path: &str) -> anyhow::Result { + let (parent, basename) = target_path + .rsplit_once('/') + .ok_or_else(|| format_err!("invalid target: {target_path}"))?; + if basename.is_empty() { + bail!("invalid target: {target_path}") + } + + // Strip the leading `.` component if present. If the basename + // contains no `.`, keep it as-is (matches the previous behaviour). + let basename_trimmed = basename.split_once('.').map_or(basename, |(_, rest)| rest); + + Ok(format!("{parent}/{basename_trimmed}")) +} + +pub(crate) struct NewTarget { + pub path: String, + pub version: u64, + /// Lowercase hex of the primary hash. + pub primary_hash: String, + /// All `(algorithm_name, hex_hash)` pairs for the target. + pub hashes: Vec<(String, String)>, + pub content: Vec, +} + +pub(crate) use cache::TargetCache; + +/// This module serves as a way to decouple the agentless logic from the rest of this crate +/// This is done for two purposes: +/// * Making review easier by allowing independent review of the RC checking alone +/// * Being able to isolate the agentless logic in it's own crate eventually so that we can reuse it +/// in bottlecap/ obs-pipeline without the rest of the code +mod cache { + use std::sync::Arc; + + use hashbrown::HashMap; + use libdd_common::MutexExt as _; + use libdd_trace_protobuf::remoteconfig::{ConfigState, TargetFileHash, TargetFileMeta}; + use std::sync::Mutex; + use tracing::warn; + + use crate::{ + fetch::{ClientTargetRef, ConfigFetcherState, FileStorage, NewTarget, StoredTargetFile}, + RemoteConfigPath, + }; + + pub(crate) struct TargetCache<'a, Storage: FileStorage> { + files: &'a Mutex, StoredTargetFile>>, + storage: &'a Storage, + expire_unused_files: bool, + } + + impl<'a, S: FileStorage> TargetCache<'a, S> { + pub(crate) fn new(state: &'a ConfigFetcherState, storage: &'a S) -> Self { + TargetCache { + files: &state.target_files_by_path, + storage, + expire_unused_files: state.expire_unused_files, + } + } + + /// Returns the TUF path strings whose `(primary_hash, len)` already matches the cache. + pub(crate) fn is_cached_batch<'b>( + &self, + candidates: impl IntoIterator, + ) -> hashbrown::HashSet<&'b str> { + let files = self.files.lock_or_panic(); + candidates + .into_iter() + .filter_map(|(path, primary_hash, len)| { + let parsed = RemoteConfigPath::try_parse(path).ok()?; + let stored = files.get(&parsed)?; + if stored.hash == primary_hash && stored.meta.length as u64 == len { + Some(path) + } else { + None + } + }) + .collect() + } + + pub(crate) fn store_batch( + &self, + targets: impl IntoIterator, + ) -> anyhow::Result<()> { + let mut files = self.files.lock_or_panic(); + for NewTarget { + path, + version, + primary_hash, + hashes, + content, + } in targets + { + let parsed_path = match RemoteConfigPath::try_parse(&path) { + Ok(p) => p, + Err(e) => { + warn!("store_batch: failed to parse remote config path {path}: {e:?}"); + continue; + } + }; + let parsed_path: Arc = Arc::new(parsed_path.into()); + let length = content.len() as i64; + let new_handle = if let Some(existing) = files.get(&parsed_path) { + self.storage + .update(&existing.handle, version, content) + .map(|()| existing.handle.clone())? + } else { + self.storage.store(version, parsed_path.clone(), content)? + }; + files.insert( + parsed_path.clone(), + StoredTargetFile { + hash: primary_hash, + state: ConfigState { + id: parsed_path.config_id.to_string(), + version, + product: parsed_path.product.to_string(), + apply_state: 2, // Acknowledged + apply_error: String::new(), + }, + meta: TargetFileMeta { + path, + length, + hashes: hashes + .into_iter() + .map(|(algorithm, hash)| TargetFileHash { algorithm, hash }) + .collect(), + }, + handle: new_handle, + expiring: false, + }, + ); + } + Ok(()) + } + + /// Evict every entry whose TUF path is not in `active_paths`. No-op when + /// `expire_unused_files` is `false`. + pub(crate) fn retain_only(&self, active_paths: &hashbrown::HashSet<&str>) { + if !self.expire_unused_files { + return; + } + self.files + .lock_or_panic() + .retain(|_, stored| active_paths.contains(stored.meta.path.as_str())); + } + + /// Collect `Arc` handles for every target in `targets`, verifying + /// stored hash and length match, and marking each entry as non-expiring. + pub(crate) fn collect_handles( + &self, + targets: &[ClientTargetRef], + ) -> anyhow::Result>> { + let mut files = self.files.lock_or_panic(); + let mut handles = Vec::with_capacity(targets.len()); + for target in targets { + let parsed = RemoteConfigPath::try_parse(&target.path).map_err(|e| { + anyhow::format_err!("collect_handles: bad path {}: {e:?}", target.path) + })?; + let stored = files.get_mut(&parsed).ok_or_else(|| { + anyhow::format_err!( + "collect_handles: path {} not found in cache after fetch", + target.path + ) + })?; + if stored.hash != target.primary_hash || stored.meta.length as u64 != target.length + { + anyhow::bail!( + "collect_handles: cache mismatch for {}: stored hash={} len={}, expected hash={} len={}", + target.path, stored.hash, stored.meta.length, + target.primary_hash, target.length + ); + } + stored.expiring = false; + handles.push(stored.handle.clone()); + } + Ok(handles) + } + + pub(crate) fn is_parseable_path(&self, path: &str) -> bool { + RemoteConfigPath::try_parse(path).is_ok() + } + } +} + +// ── Debug helpers: render `raw: Vec` fields as JSON ──────────────────── + +struct RawJson<'a>(&'a [u8]); + +impl fmt::Debug for RawJson<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let RawJson(bytes) = self; + match serde_json::from_slice::(bytes) { + Ok(v) => write!(f, "{v:#}"), + Err(_) => write!(f, "<{} non-JSON bytes>", bytes.len()), + } + } +} + +struct DebugTopMeta<'a>(&'a remoteconfig::TopMeta); + +impl fmt::Debug for DebugTopMeta<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::TopMeta { version, raw } = self.0; + f.debug_struct("TopMeta") + .field("version", version) + .field("raw", &RawJson(raw)) + .finish() + } +} + +struct DebugDelegatedMeta<'a>(&'a remoteconfig::DelegatedMeta); + +impl fmt::Debug for DebugDelegatedMeta<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::DelegatedMeta { version, role, raw } = self.0; + f.debug_struct("DelegatedMeta") + .field("version", version) + .field("role", role) + .field("raw", &RawJson(raw)) + .finish() + } +} + +struct DebugFile<'a>(&'a remoteconfig::File); + +impl fmt::Debug for DebugFile<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::File { path, raw } = self.0; + f.debug_struct("File") + .field("path", path) + .field("raw", &RawJson(raw)) + .finish() + } +} + +struct DebugConfigMetas<'a>(&'a remoteconfig::ConfigMetas); + +impl fmt::Debug for DebugConfigMetas<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::ConfigMetas { + roots, + timestamp, + snapshot, + top_targets, + delegated_targets, + } = self.0; + f.debug_struct("ConfigMetas") + .field("roots", &roots.iter().map(DebugTopMeta).collect::>()) + .field("timestamp", ×tamp.as_ref().map(DebugTopMeta)) + .field("snapshot", &snapshot.as_ref().map(DebugTopMeta)) + .field("top_targets", &top_targets.as_ref().map(DebugTopMeta)) + .field( + "delegated_targets", + &delegated_targets + .iter() + .map(DebugDelegatedMeta) + .collect::>(), + ) + .finish() + } +} + +struct DebugDirectorMetas<'a>(&'a remoteconfig::DirectorMetas); + +impl fmt::Debug for DebugDirectorMetas<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::DirectorMetas { + roots, + timestamp, + snapshot, + targets, + } = &self.0; + f.debug_struct("DirectorMetas") + .field("roots", &roots.iter().map(DebugTopMeta).collect::>()) + .field("timestamp", ×tamp.as_ref().map(DebugTopMeta)) + .field("snapshot", &snapshot.as_ref().map(DebugTopMeta)) + .field("targets", &targets.as_ref().map(DebugTopMeta)) + .finish() + } +} + +struct DebugLatestConfigsResponse<'a>(&'a remoteconfig::LatestConfigsResponse); + +impl fmt::Debug for DebugLatestConfigsResponse<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::LatestConfigsResponse { + config_metas, + director_metas, + target_files, + } = &self.0; + f.debug_struct("LatestConfigsResponse") + .field("config_metas", &config_metas.as_ref().map(DebugConfigMetas)) + .field( + "director_metas", + &director_metas.as_ref().map(DebugDirectorMetas), + ) + .field( + "target_files", + &target_files.iter().map(DebugFile).collect::>(), + ) + .finish() + } +} + +/// Returns a value that implements [`fmt::Debug`] for [`remoteconfig::LatestConfigsResponse`], +/// rendering every `raw` byte field as a parsed JSON value instead of a raw byte array. +pub fn debug_latest_configs_response( + resp: &remoteconfig::LatestConfigsResponse, +) -> impl fmt::Debug + '_ { + DebugLatestConfigsResponse(resp) +} + +#[cfg(test)] +mod tests { + use super::trim_hash_target_path; + use super::Site; + + #[test] + fn strips_hash_prefix() { + assert_eq!( + trim_hash_target_path("datadog/2/APM_TRACING/abcd/deadbeef.config").unwrap(), + "datadog/2/APM_TRACING/abcd/config" + ); + } + + #[test] + fn no_hash_prefix_is_kept() { + assert_eq!( + trim_hash_target_path("datadog/2/APM_TRACING/abcd/config").unwrap(), + "datadog/2/APM_TRACING/abcd/config" + ); + } + + #[test] + fn backslash_is_not_a_separator() { + // Windows-style separators must NOT be treated as path separators. + // The whole string is the basename here. + assert!(trim_hash_target_path(r"datadog\2\foo.bar").is_err()); + } + + #[test] + fn empty_or_no_slash_is_error() { + assert!(trim_hash_target_path("").is_err()); + assert!(trim_hash_target_path("deadbeef.config").is_err()); + } + + #[test] + fn trailing_slash_is_error() { + assert!(trim_hash_target_path("datadog/2/foo/").is_err()); + } + + #[test] + fn test_compute_backoff() { + use super::compute_backoff; + use std::time::Duration; + + assert_eq!(compute_backoff(0), None); + assert_eq!(compute_backoff(1), None); + + let b2 = compute_backoff(2).unwrap(); + assert!((Duration::from_secs(30)..=Duration::from_secs(60)).contains(&b2)); + + let b3 = compute_backoff(3).unwrap(); + assert!((Duration::from_secs(60)..=Duration::from_secs(120)).contains(&b3)); + + assert_eq!(compute_backoff(4), Some(Duration::from_secs(120))); + assert_eq!(compute_backoff(42), Some(Duration::from_secs(120))); + } + + #[test] + fn test_target_matches_pattern() { + use super::target_matches_pattern as m; + + // Real Datadog delegation pattern. + assert!(m( + "datadog/556989/APM_TRACING/abc/def", + "datadog/*/APM_TRACING/*/*" + )); + + // Wrong product segment. + assert!(!m( + "datadog/556989/ASM_DD/abc/def", + "datadog/*/APM_TRACING/*/*" + )); + + // Extra path segment must not match (`*` doesn't cross `/`). + assert!(!m( + "datadog/x/y/APM_TRACING/abc/def", + "datadog/*/APM_TRACING/*/*" + )); + + // Missing path segment. + assert!(!m( + "datadog/556989/APM_TRACING/abc", + "datadog/*/APM_TRACING/*/*" + )); + + // Employee-prefix delegation. + assert!(m("employee/ASM_DD/abc/def", "employee/ASM_DD/*/*")); + assert!(!m("employee/CWS_DD/abc/def", "employee/ASM_DD/*/*")); + + // Partial-segment wildcards. + assert!(super::segment_matches("foo*bar", "foo123bar")); + assert!(super::segment_matches("foo*", "foobar")); + assert!(super::segment_matches("*bar", "foobar")); + assert!(super::segment_matches( + "ba*bar k*g of the *elepha*ts*", + "babar king of the elephants" + )); + assert!(!super::segment_matches("foo*bar", "fobar")); + assert!(!super::segment_matches(" *foobar**", "fobar")); + // `*` does not cross `/`. + assert!(!m("foo/bar", "foo*bar")); + } + + #[test] + fn test_site_from_host() { + assert_eq!(Site::from_host("config.datadoghq.com"), Site::Prod); + assert_eq!(Site::from_host("config.us3.datadoghq.com"), Site::Prod); + assert_eq!(Site::from_host("config.datadoghq.eu"), Site::Prod); + assert_eq!(Site::from_host("config.datad0g.com"), Site::Staging); + assert_eq!(Site::from_host("datad0g.com"), Site::Staging); + assert_eq!(Site::from_host("config.ddog-gov.com"), Site::Gov); + assert_eq!(Site::from_host("config.foo.ddog-gov.com"), Site::Gov); + } +} + +#[cfg(test)] +mod integration_tests; diff --git a/libdd-remote-config/src/fetch/agentless/integration_tests.rs b/libdd-remote-config/src/fetch/agentless/integration_tests.rs new file mode 100644 index 0000000000..205d66df09 --- /dev/null +++ b/libdd-remote-config/src/fetch/agentless/integration_tests.rs @@ -0,0 +1,908 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Mirrors the datadog-agent uptane +//! `client_test.go` harness (generate signed config + director repos, feed a +//! `LatestConfigsResponse` to the client), but drives libdatadog's +//! `fetch_config`/`apply` path through a mock HTTP capability. +//! +//! Test root rotation, and different input shapes +#![allow(clippy::unwrap_used)] + +use super::*; +use crate::fetch::{ConfigFetcherState, ConfigInvariants, FileStorage}; +use crate::RemoteConfigPath; +use libdd_capabilities::http::{HttpClientCapability, HttpError}; +use libdd_capabilities::maybe_send::MaybeSend; +use std::collections::VecDeque; +use std::future::Future; +use std::sync::{Arc, Mutex}; +use tuf::crypto::{Ed25519PrivateKey, HashAlgorithm, PrivateKey}; +use tuf::database::Database; +use tuf::interchange::Json; +use tuf::metadata::{ + Delegation, MetadataDescription, MetadataPath, RawSignedMetadataSet, SignedMetadata, + SnapshotMetadata, TargetPath, TargetsMetadataBuilder, TimestampMetadataBuilder, +}; +use tuf::repo_builder::RepoBuilder; +use tuf::repository::EphemeralRepository; + +// ---- mock HTTP capability ------------------------------------------------ + +#[derive(Clone, Debug)] +struct MockHttp { + responses: Arc>>>, + requests: Arc>>, + /// UUID returned for GET /api/v0.1/org. Some tests want to change it mid-run + /// (e.g. serve a mismatching UUID) so this is a Mutex, not an atomic snapshot. + org_uuid: Arc>, + /// Count of GET /api/v0.1/org requests observed, used to assert the + /// concurrent prefetch (poll 1) and the lazy re-fetch (post root-rotation). + org_requests: Arc>, +} + +impl MockHttp { + fn new() -> Self { + Self { + responses: Arc::new(Mutex::new(VecDeque::new())), + requests: Arc::new(Mutex::new(Vec::new())), + org_uuid: Arc::new(Mutex::new(String::new())), + org_requests: Arc::new(Mutex::new(0)), + } + } + + fn push(&self, resp: &remoteconfig::LatestConfigsResponse) { + self.responses + .lock() + .unwrap() + .push_back(resp.encode_to_vec()); + } + + fn request_at(&self, i: usize) -> remoteconfig::LatestConfigsRequest { + self.requests.lock().unwrap()[i].clone() + } + + fn set_org_uuid(&self, uuid: &str) { + *self.org_uuid.lock().unwrap() = uuid.to_string(); + } + + fn org_request_count(&self) -> u32 { + *self.org_requests.lock().unwrap() + } +} + +impl HttpClientCapability for MockHttp { + fn new_client() -> Self { + Self::new() + } + + #[allow(clippy::manual_async_fn)] + fn request( + &self, + req: http::Request, + ) -> impl Future, HttpError>> + MaybeSend { + let responses = self.responses.clone(); + let requests = self.requests.clone(); + let org_uuid = self.org_uuid.clone(); + let org_requests = self.org_requests.clone(); + // Capture the path off the request before moving the body. + let path = req.uri().path().to_owned(); + async move { + if path == "/api/v0.1/org" { + *org_requests.lock().unwrap() += 1; + let resp = remoteconfig::OrgDataResponse { + uuid: org_uuid.lock().unwrap().clone(), + }; + return Ok(http::Response::builder() + .status(200) + .body(Bytes::from(resp.encode_to_vec())) + .unwrap()); + } + let body = req.into_body(); + if let Ok(parsed) = remoteconfig::LatestConfigsRequest::decode(body) { + requests.lock().unwrap().push(parsed); + } + let bytes = responses + .lock() + .unwrap() + .pop_front() + .expect("mock http: no queued response"); + Ok(http::Response::builder() + .status(200) + .body(Bytes::from(bytes)) + .unwrap()) + } + } +} + +// ---- no-op file storage -------------------------------------------------- + +#[derive(Default)] +struct NoopStorage; + +impl FileStorage for NoopStorage { + type StoredFile = (); + + fn store( + &self, + _version: u64, + _path: Arc, + _contents: Vec, + ) -> anyhow::Result> { + Ok(Arc::new(())) + } + + fn update( + &self, + _file: &Arc, + _version: u64, + _contents: Vec, + ) -> anyhow::Result<()> { + Ok(()) + } +} + +fn test_state() -> ConfigFetcherState<()> { + ConfigFetcherState::new(ConfigInvariants { + language: "test".to_string(), + tracer_version: "0.0.0".to_string(), + endpoint: Endpoint::from_slice("http://localhost/"), + agentless: None, + }) +} + +// ---- TUF repo generation (mirrors uptane client_test.go) ----------------- + +fn new_key() -> Ed25519PrivateKey { + Ed25519PrivateKey::from_pkcs8(&Ed25519PrivateKey::pkcs8().unwrap()).unwrap() +} + +/// Build a fresh v1 repo (root/targets/snapshot/timestamp all v1, empty +/// targets, consistent_snapshot=true) signed entirely by `key`. +async fn build_v1(key: &Ed25519PrivateKey) -> RawSignedMetadataSet { + let mut repo = EphemeralRepository::::new(); + RepoBuilder::create(&mut repo) + .trusted_root_keys(&[key]) + .trusted_targets_keys(&[key]) + .trusted_snapshot_keys(&[key]) + .trusted_timestamp_keys(&[key]) + .commit() + .await + .unwrap() +} + +/// Rotate only the root (v1 -> v2), keeping the same keys. rust-tuf's +/// `update_root` purges all non-root trusted metadata on this bump, which is +/// what triggers the top-targets re-fetch the stuck test exercises. +async fn rotate_root( + key: &Ed25519PrivateKey, + prev: &RawSignedMetadataSet, +) -> RawSignedMetadataSet { + let db = Database::::from_trusted_metadata(prev).unwrap(); + let mut repo = EphemeralRepository::::new(); + RepoBuilder::from_database(&mut repo, &db) + .trusted_root_keys(&[key]) + .trusted_targets_keys(&[key]) + .trusted_snapshot_keys(&[key]) + .trusted_timestamp_keys(&[key]) + .stage_root() + .unwrap() + .commit() + .await + .unwrap() +} + +fn meta_version(raw: &[u8]) -> u64 { + let v: serde_json::Value = serde_json::from_slice(raw).unwrap(); + v["signed"]["version"].as_u64().unwrap() +} + +fn top(raw: &[u8]) -> remoteconfig::TopMeta { + remoteconfig::TopMeta { + version: meta_version(raw), + raw: raw.to_vec(), + } +} + +fn director_metas(set: &RawSignedMetadataSet) -> remoteconfig::DirectorMetas { + remoteconfig::DirectorMetas { + roots: vec![top(set.root().unwrap().as_bytes())], + timestamp: Some(top(set.timestamp().unwrap().as_bytes())), + snapshot: Some(top(set.snapshot().unwrap().as_bytes())), + targets: Some(top(set.targets().unwrap().as_bytes())), + } +} + +/// Build a `LatestConfigsResponse` from raw config metadata plus a director set. +fn response( + config_roots: &[&[u8]], + config_timestamp: &[u8], + config_snapshot: &[u8], + config_top_targets: Option<&[u8]>, + delegated: Vec, + director: &RawSignedMetadataSet, +) -> remoteconfig::LatestConfigsResponse { + remoteconfig::LatestConfigsResponse { + config_metas: Some(remoteconfig::ConfigMetas { + roots: config_roots.iter().map(|r| top(r)).collect(), + timestamp: Some(top(config_timestamp)), + snapshot: Some(top(config_snapshot)), + top_targets: config_top_targets.map(top), + delegated_targets: delegated, + }), + director_metas: Some(director_metas(director)), + target_files: vec![], + } +} + +/// Construct a fetcher wired to a mock HTTP client and pinned to the given +/// root bytes (bypassing `AgentlessFetcher::new`, whose `C::new_client()` +/// would discard our pre-seeded mock). +async fn fetcher( + http: MockHttp, + config_root: Vec, + director_root: Vec, +) -> AgentlessFetcher { + AgentlessFetcher { + endpoint: Endpoint { + timeout_ms: 30_000, + ..Endpoint::from_slice("http://localhost/") + }, + http, + director_client: TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(director_root.clone()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await + .unwrap(), + config_client: TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(config_root.clone()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await + .unwrap(), + config_root_bytes: Cow::Owned(config_root), + director_root_bytes: Cow::Owned(director_root), + last_config_top_targets: None, + org_uuid: None, + org_data_prefetched: false, + hostname: "test-host".to_string(), + agent_uuid_override: Some("test-uuid".to_string()), + products: HashSet::new(), + opaque_backend_state: Vec::new(), + refresh_interval: Duration::from_secs(60), + consecutive_failures: 0, + } +} + +fn dummy_client() -> remoteconfig::Client { + remoteconfig::Client { + products: vec!["APM_TRACING".to_string()], + ..Default::default() + } +} + +fn config_root_version(f: &AgentlessFetcher) -> u64 { + f.config_client.database().trusted_root().version() +} + +fn config_snapshot_version(f: &AgentlessFetcher) -> Option { + f.config_client + .database() + .trusted_snapshot() + .map(|s| s.version()) +} + +// ---- tests --------------------------------------------------------------- + +/// incident-45734: a config **root rotation** where the backend omits the +/// (unchanged) top-targets must still converge. Before the fix the wipe drops +/// the top-targets and `update()` is stuck; the cache re-serves them. +#[tokio::test] +async fn test_root_rotation_without_top_targets_still_converges() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let cfg2 = rotate_root(&config_key, &cfg1).await; // config root v1 -> v2 + let dir1 = build_v1(&director_key).await; + + let http = MockHttp::new(); + // Poll 1: full config metadata. + http.push(&response( + &[cfg1.root().unwrap().as_bytes()], + cfg1.timestamp().unwrap().as_bytes(), + cfg1.snapshot().unwrap().as_bytes(), + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + // Poll 2: config ROOT rotated (v2), top-targets version unchanged so the + // backend sends NONE; reuse the v1 timestamp/snapshot. + http.push(&response( + &[cfg2.root().unwrap().as_bytes()], + cfg1.timestamp().unwrap().as_bytes(), + cfg1.snapshot().unwrap().as_bytes(), + None, + vec![], + &dir1, + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + // Poll 1 succeeds and advances the config DB to root v1 / snapshot v1. + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 1); + + // Poll 2 (root rotation, no top-targets) must still converge. + f.fetch_config(dummy_client(), &cache) + .await + .expect("root rotation with omitted top-targets must converge"); + assert_eq!(config_root_version(&f), 2); + + // Step 10: reported versions always match the live trusted DB. + let req1 = http.request_at(0); + assert_eq!(req1.current_config_snapshot_version, 0); + assert_eq!(req1.current_config_root_version, 1); + assert_eq!(req1.current_director_root_version, 1); + let req2 = http.request_at(1); + // After poll 1 the DB advanced: snapshot v1, config root still v1. + assert_eq!( + req2.current_config_snapshot_version, + meta_version(cfg1.snapshot().unwrap().as_bytes()) + ); + assert_eq!( + req2.current_config_root_version, + meta_version(cfg1.root().unwrap().as_bytes()) + ); +} + +/// D-F1: an `apply()` that fails *after* advancing the config trusted DB must +/// leave the fetcher recoverable. The reset rebuilds the clients from the +/// pinned roots, so the next poll reports the clean (embedded) versions and +/// converges — no stuck from a partially-advanced trusted DB. +#[tokio::test] +async fn test_apply_error_resets_and_recovers() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let cfg2 = rotate_root(&config_key, &cfg1).await; + let dir1 = build_v1(&director_key).await; + + let good = |top_targets: Option<&[u8]>, roots: &[&[u8]]| { + response( + roots, + cfg1.timestamp().unwrap().as_bytes(), + cfg1.snapshot().unwrap().as_bytes(), + top_targets, + vec![], + &dir1, + ) + }; + + let http = MockHttp::new(); + // Poll 1: good, advances to config root v1. + http.push(&good( + Some(cfg1.targets().unwrap().as_bytes()), + &[cfg1.root().unwrap().as_bytes()], + )); + // Poll 2: config root rotates to v2 (config update succeeds and advances + // the trusted root), then a garbage delegated-targets blob makes apply() + // fail *after* the advance. + let mut bad = good( + Some(cfg1.targets().unwrap().as_bytes()), + &[cfg2.root().unwrap().as_bytes()], + ); + bad.config_metas.as_mut().unwrap().delegated_targets = vec![remoteconfig::DelegatedMeta { + version: 1, + role: "APM_TRACING".to_string(), + raw: b"not valid tuf metadata".to_vec(), + }]; + http.push(&bad); + // Poll 3: good again — must recover. + http.push(&good( + Some(cfg1.targets().unwrap().as_bytes()), + &[cfg1.root().unwrap().as_bytes()], + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + // Poll 1 ok. + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 1); + assert_eq!(config_snapshot_version(&f), Some(1)); + // A fully successful poll leaves the backoff counter at 0. + assert_eq!(f.consecutive_failures(), 0); + assert_eq!(f.next_backoff(), None); + + // Poll 2 fails (after the config root advanced to v2) and resets. + assert!(f.fetch_config(dummy_client(), &cache).await.is_err()); + // Reset rebuilt the config client from the pinned root: back to v1, no snapshot. + assert_eq!(config_root_version(&f), 1); + assert_eq!(config_snapshot_version(&f), None); + assert!(f.opaque_backend_state.is_empty()); + assert!(f.products.is_empty()); + assert!(f.last_config_top_targets.is_none()); + // E-1: a successful HTTP fetch that fails `apply()` (verification) must still + // count as a failed poll, so the client backs off during sustained + // verification failures instead of hot-looping. + assert_eq!(f.consecutive_failures(), 1); + + // Poll 3 recovers. + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 1); + assert_eq!(config_snapshot_version(&f), Some(1)); + // A fully successful poll clears the counter again. + assert_eq!(f.consecutive_failures(), 0); + assert_eq!(f.next_backoff(), None); + + // The post-reset poll reported the clean embedded versions, matching the + // live trusted DB (snapshot 0, root v1) — not the advanced v2. + let req3 = http.request_at(2); + assert_eq!(req3.current_config_snapshot_version, 0); + assert_eq!(req3.current_config_root_version, 1); + assert_eq!(req3.current_director_root_version, 1); +} + +/// Build a config repo (v1) that authorizes both `known_path` and +/// `unknown_path` through a single delegated role `role_name` whose glob +/// `paths` cover both products. Returns the signed config metadata set plus the +/// raw delegated-targets blob (to feed as `DelegatedMeta`). +async fn build_config_with_delegation( + config_key: &Ed25519PrivateKey, + product_key: &Ed25519PrivateKey, + role_name: &str, + entries: &[(&str, &[u8])], + glob_paths: &[&str], + target_hashes: &[HashAlgorithm], +) -> (RawSignedMetadataSet, Vec) { + // Delegated targets blob: describes every authorized (path, content). + let mut builder = TargetsMetadataBuilder::new(); + for (path, content) in entries { + builder = builder + .insert_target_from_slice( + TargetPath::new((*path).to_string()).unwrap(), + content, + target_hashes, + ) + .unwrap(); + } + let delegated = builder.signed::(product_key).unwrap(); + let raw_delegated = delegated.to_raw().unwrap().as_bytes().to_vec(); + + // Top-level config targets: delegate the glob paths to `role_name`. + let mut delegation = Delegation::builder(MetadataPath::new(role_name.to_string()).unwrap()) + .key(product_key.public()); + for g in glob_paths { + delegation = delegation.delegate_path(TargetPath::new((*g).to_string()).unwrap()); + } + let delegation = delegation.build().unwrap(); + + let role_path = MetadataPath::new(role_name.to_string()).unwrap(); + let delegated_desc = + MetadataDescription::from_slice(&raw_delegated, 1, &[HashAlgorithm::Sha256]).unwrap(); + + let mut repo = EphemeralRepository::::new(); + let set = RepoBuilder::create(&mut repo) + .trusted_root_keys(&[config_key]) + .trusted_targets_keys(&[config_key]) + .trusted_snapshot_keys(&[config_key]) + .trusted_timestamp_keys(&[config_key]) + .stage_root() + .unwrap() + .add_delegation_key(product_key.public().clone()) + .add_delegation_role(delegation) + .stage_targets() + .unwrap() + .stage_snapshot_with_builder(|builder| { + builder.insert_metadata_description(role_path.clone(), delegated_desc.clone()) + }) + .unwrap() + .commit() + .await + .unwrap(); + + (set, raw_delegated) +} + +/// Build a director repo (v1) that announces every `(path, content)` entry as a +/// top-level target (sha256), matching the config authorization. +async fn build_director_with_targets( + director_key: &Ed25519PrivateKey, + entries: &[(&str, &[u8])], + target_hashes: &[HashAlgorithm], +) -> RawSignedMetadataSet { + let mut repo = EphemeralRepository::::new(); + let mut builder = RepoBuilder::create(&mut repo) + .trusted_root_keys(&[director_key]) + .trusted_targets_keys(&[director_key]) + .trusted_snapshot_keys(&[director_key]) + .trusted_timestamp_keys(&[director_key]) + .stage_root_if_necessary() + .unwrap() + .target_hash_algorithms(target_hashes); + for (path, content) in entries { + builder = builder + .add_target( + TargetPath::new((*path).to_string()).unwrap(), + futures_util::io::Cursor::new(content.to_vec()), + ) + .await + .unwrap(); + } + builder.stage_targets().unwrap().commit().await.unwrap() +} + +/// E-F1 / G-F1: a director target for a product the closed `RemoteConfigProduct` +/// enum does not know must not fail the fetch of the other, known targets. +/// +/// The cache owns the parsing rules (`TargetCache::is_parseable_path`); `apply()` +/// consults it to drop unparseable/unknown-product targets before they reach +/// `active_targets`, so `collect_handles` never sees a path it can't serve. +#[tokio::test] +async fn test_unknown_product_target_is_not_stuck_known_targets() { + let config_key = new_key(); + let product_key = new_key(); + let director_key = new_key(); + + let known_path = "datadog/2/APM_TRACING/cfgid/config"; + let unknown_path = "datadog/2/BRAND_NEW_PRODUCT/cfgid/config"; + let known_content: &[u8] = b"known apm config payload"; + let unknown_content: &[u8] = b"brand new product payload"; + + let entries: &[(&str, &[u8])] = &[(known_path, known_content), (unknown_path, unknown_content)]; + + // Config authorizes BOTH paths (so `verify_director_against_config` passes); + // the divergence is purely that libdatadog's product enum can't parse the + // second one. + let (cfg, raw_delegated) = build_config_with_delegation( + &config_key, + &product_key, + "APM_TRACING", + entries, + &[ + "datadog/*/APM_TRACING/*/*", + "datadog/*/BRAND_NEW_PRODUCT/*/*", + ], + &[HashAlgorithm::Sha256], + ) + .await; + let dir = build_director_with_targets(&director_key, entries, &[HashAlgorithm::Sha256]).await; + + let resp = delegated_response(&cfg, raw_delegated, "APM_TRACING", &dir, entries); + + let http = MockHttp::new(); + http.push(&resp); + + let mut f = fetcher( + http.clone(), + cfg.root().unwrap().as_bytes().to_vec(), + dir.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let res = f + .fetch_config(dummy_client(), &cache) + .await + .expect("a config-authorized unknown-product target must not stuck the fetch"); + + // The cache knows it cannot serve the unknown-product path. + assert!(cache.is_parseable_path(known_path)); + assert!(!cache.is_parseable_path(unknown_path)); + + // `apply()` filtered it out, so only the known target is active — the unknown + // product never reaches `active_targets`/`collect_handles`. + let returned: Vec<&str> = res.targets.iter().map(|t| t.path.as_str()).collect(); + assert_eq!( + returned, + vec![known_path], + "only the known-product target should be active" + ); + + // The active batch is fully servable: `collect_handles` succeeds (no stuck). + let handles = cache + .collect_handles(&res.targets) + .expect("active batch must not stuck collect_handles"); + assert_eq!(handles.len(), 1, "only the known target should be served"); +} + +/// Assemble a `LatestConfigsResponse` from a config set + its raw delegated +/// blob, a director set, and the `(path, content)` entries served as files. +fn delegated_response( + cfg: &RawSignedMetadataSet, + raw_delegated: Vec, + role_name: &str, + dir: &RawSignedMetadataSet, + entries: &[(&str, &[u8])], +) -> remoteconfig::LatestConfigsResponse { + remoteconfig::LatestConfigsResponse { + config_metas: Some(remoteconfig::ConfigMetas { + roots: vec![top(cfg.root().unwrap().as_bytes())], + timestamp: Some(top(cfg.timestamp().unwrap().as_bytes())), + snapshot: Some(top(cfg.snapshot().unwrap().as_bytes())), + top_targets: Some(top(cfg.targets().unwrap().as_bytes())), + delegated_targets: vec![remoteconfig::DelegatedMeta { + version: meta_version(&raw_delegated), + role: role_name.to_string(), + raw: raw_delegated, + }], + }), + director_metas: Some(director_metas(dir)), + target_files: entries + .iter() + .map(|(path, content)| remoteconfig::File { + path: (*path).to_string(), + raw: content.to_vec(), + }) + .collect(), + } +} + +/// A-F1 (libdd #14): `verify_director_against_config` must require the director +/// and config hash sets to be *exactly equal*. Here the director publishes +/// sha256+sha512 for a target while config pins only sha256. The old +/// overlap-only check (shared algos agree) would accept this — letting the +/// director assert an arbitrary sha512 digest config never authorized — so the +/// whole `apply()` must now fail. +#[tokio::test] +async fn test_director_hash_superset_is_rejected() { + let config_key = new_key(); + let product_key = new_key(); + let director_key = new_key(); + + let path = "datadog/2/APM_TRACING/cfgid/config"; + let content: &[u8] = b"apm config payload"; + let entries: &[(&str, &[u8])] = &[(path, content)]; + + // Config pins sha256 only. + let (cfg, raw_delegated) = build_config_with_delegation( + &config_key, + &product_key, + "APM_TRACING", + entries, + &["datadog/*/APM_TRACING/*/*"], + &[HashAlgorithm::Sha256], + ) + .await; + // Director publishes a superset: sha256 + sha512 (digests still correct for + // the content, so only the *set* differs). + let dir = build_director_with_targets( + &director_key, + entries, + &[HashAlgorithm::Sha256, HashAlgorithm::Sha512], + ) + .await; + + let resp = delegated_response(&cfg, raw_delegated, "APM_TRACING", &dir, entries); + let http = MockHttp::new(); + http.push(&resp); + + let mut f = fetcher( + http.clone(), + cfg.root().unwrap().as_bytes().to_vec(), + dir.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let Err(err) = f.fetch_config(dummy_client(), &cache).await else { + panic!("director hash superset must be rejected (exact-equality)"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("hash set mismatch"), + "expected a hash-set mismatch error, got: {msg}" + ); +} + +/// Build a config snapshot + timestamp pair whose signed snapshot carries +/// `custom.org_uuid = uuid`. Reuses the (version, expires, meta) already +/// signed inside `cfg.snapshot()`, adds the custom field, and re-signs with +/// `config_key`. The timestamp is rebuilt from the new snapshot so its +/// (length, hash) description stays consistent. +fn config_snapshot_with_org_uuid( + cfg: &RawSignedMetadataSet, + config_key: &Ed25519PrivateKey, + uuid: &str, +) -> (Vec, Vec) { + // Grab the `signed` object out of the raw snapshot bytes, inject + // `custom.org_uuid`, then re-hydrate a `SnapshotMetadata` via serde (the + // rust-tuf shim flattens unknown top-level fields into `additional_fields`, + // so our injected `custom` object lands in the right place). + let mut signed: serde_json::Value = + serde_json::from_slice(cfg.snapshot().unwrap().as_bytes()).unwrap(); + signed + .get_mut("signed") + .unwrap() + .as_object_mut() + .unwrap() + .insert( + "custom".to_string(), + serde_json::json!({ "org_uuid": uuid }), + ); + let new_snap: SnapshotMetadata = serde_json::from_value(signed["signed"].clone()).unwrap(); + let signed_snap = SignedMetadata::::new(&new_snap, config_key).unwrap(); + let raw_snap = signed_snap.to_raw().unwrap().as_bytes().to_vec(); + + let signed_ts = + TimestampMetadataBuilder::from_snapshot::(&signed_snap, &[HashAlgorithm::Sha256]) + .unwrap() + .signed::(config_key) + .unwrap(); + let raw_ts = signed_ts.to_raw().unwrap().as_bytes().to_vec(); + + (raw_snap, raw_ts) +} + +/// First-poll happy path: the concurrent org-data prefetch returns the same +/// UUID that the snapshot pins, so the fetch converges and exactly one org +/// request was issued. +#[tokio::test] +async fn test_org_uuid_match_via_concurrent_prefetch() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let dir1 = build_v1(&director_key).await; + let (snap_bytes, ts_bytes) = config_snapshot_with_org_uuid(&cfg1, &config_key, "ORG-1"); + + let http = MockHttp::new(); + http.set_org_uuid("ORG-1"); + http.push(&response( + &[cfg1.root().unwrap().as_bytes()], + &ts_bytes, + &snap_bytes, + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + f.fetch_config(dummy_client(), &cache) + .await + .expect("matching org UUID must not fail the poll"); + + // Exactly one org request on the first poll (concurrent prefetch). + assert_eq!(http.org_request_count(), 1); +} + +/// Mismatching org UUID must fail the poll AND reset the fetcher, so a +/// subsequent poll reports the clean (embedded) versions. +#[tokio::test] +async fn test_org_uuid_mismatch_fails_and_resets() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let dir1 = build_v1(&director_key).await; + let (snap_bytes, ts_bytes) = config_snapshot_with_org_uuid(&cfg1, &config_key, "ORG-EXPECTED"); + + let http = MockHttp::new(); + http.set_org_uuid("ORG-OTHER"); + http.push(&response( + &[cfg1.root().unwrap().as_bytes()], + &ts_bytes, + &snap_bytes, + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let Err(err) = f.fetch_config(dummy_client(), &cache).await else { + panic!("mismatching org UUID must fail the poll"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("org UUID"), + "expected an org-UUID mismatch error, got: {msg}" + ); + + // Reset restored the pinned-root state. + assert_eq!(config_root_version(&f), 1); + assert_eq!(config_snapshot_version(&f), None); + // The one-shot prefetch flag was rearmed, so a fresh poll would prefetch again. + assert!(!f.org_data_prefetched); + assert!(f.org_uuid.is_none()); +} + +/// A config-root rotation invalidates the pinned UUID (it is keyed by the +/// config trusted-root version), forcing a fresh /api/v0.1/org fetch on the +/// next poll. +#[tokio::test] +async fn test_org_uuid_refetched_on_root_rotation() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let cfg2 = rotate_root(&config_key, &cfg1).await; // config root v1 -> v2 + let dir1 = build_v1(&director_key).await; + let (snap_bytes, ts_bytes) = config_snapshot_with_org_uuid(&cfg1, &config_key, "ORG-1"); + + let http = MockHttp::new(); + http.set_org_uuid("ORG-1"); + // Poll 1: config root v1, snapshot pins ORG-1. + http.push(&response( + &[cfg1.root().unwrap().as_bytes()], + &ts_bytes, + &snap_bytes, + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + // Poll 2: config root rotates to v2; same signed snapshot/timestamp + // (rust-tuf purges non-root metadata on rotation, so the backend has to + // resend them). The pinned UUID binding keyed at root v1 no longer + // matches root v2 and forces a fresh /api/v0.1/org fetch. + http.push(&response( + &[cfg2.root().unwrap().as_bytes()], + &ts_bytes, + &snap_bytes, + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 1); + assert_eq!(http.org_request_count(), 1); // concurrent prefetch + + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 2); + // A second org fetch happened lazily because the config root rotated. + assert_eq!(http.org_request_count(), 2); +} diff --git a/libdd-remote-config/src/fetch/fetcher.rs b/libdd-remote-config/src/fetch/fetcher.rs index e43aa23b43..005865cd22 100644 --- a/libdd-remote-config/src/fetch/fetcher.rs +++ b/libdd-remote-config/src/fetch/fetcher.rs @@ -1,6 +1,9 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#[cfg(feature = "agentless")] +use super::agentless; + use crate::targets::{Root, TargetsList}; use crate::{RemoteConfigCapabilities, RemoteConfigPath, RemoteConfigProduct, Target}; use base64::Engine; @@ -51,14 +54,26 @@ pub struct ConfigInvariants { pub language: String, pub tracer_version: String, pub endpoint: Endpoint, + #[cfg(feature = "agentless")] + /// Enables and configures agentless mode. If some the fetcher will + /// talk directly to the RC backend + pub agentless: Option, + #[cfg(not(feature = "agentless"))] + pub agentless: Option, +} + +impl ConfigInvariants { + pub fn agentless_enabled(&self) -> bool { + self.agentless.is_some() + } } -struct StoredTargetFile { - hash: String, - handle: Arc, - state: ConfigState, - meta: TargetFileMeta, - expiring: bool, +pub(crate) struct StoredTargetFile { + pub(crate) hash: String, + pub(crate) handle: Arc, + pub(crate) state: ConfigState, + pub(crate) meta: TargetFileMeta, + pub(crate) expiring: bool, } pub enum ConfigApplyState { @@ -105,7 +120,7 @@ impl ConfigProductCapabilities { } pub struct ConfigFetcherState { - target_files_by_path: Mutex, StoredTargetFile>>, + pub(crate) target_files_by_path: Mutex, StoredTargetFile>>, pub invariants: ConfigInvariants, endpoint: Endpoint, pub expire_unused_files: bool, @@ -154,10 +169,36 @@ impl ConfigFetcherFilesLock<'_, S> { impl ConfigFetcherState { pub fn new(invariants: ConfigInvariants) -> Self { + let (endpoint, agentless) = match &invariants.agentless { + Some(agentless_cfg) => { + #[cfg(feature = "agentless")] + match ( + agentless::make_agentless_configs_endpoint(&invariants.endpoint), + agentless_cfg.hostname.is_empty(), + ) { + (Some(e), false) => (e, Some(agentless_cfg.clone())), + (Some(_), true) => { + warn!("rc_config_fetcher: agentless enabled but the hostname is empty. Downgrading to agent endpoint"); + (make_agent_configs_endpoint(&invariants.endpoint), None) + } + (None, _) => { + warn!("rc_config_fetcher: agentless enabled but the endpoint is invalid. Downgrading to agent endpoint"); + (make_agent_configs_endpoint(&invariants.endpoint), None) + } + } + + #[cfg(not(feature = "agentless"))] + match *agentless_cfg {} + } + None => (make_agent_configs_endpoint(&invariants.endpoint), None), + }; ConfigFetcherState { target_files_by_path: Default::default(), - endpoint: get_agent_configs_endpoint(&invariants.endpoint), - invariants, + endpoint, + invariants: ConfigInvariants { + agentless, + ..invariants + }, expire_unused_files: true, } } @@ -201,9 +242,17 @@ impl ConfigFetcherState { } } +#[allow(clippy::large_enum_variant)] +enum FetcherMode { + Agent, + #[cfg(feature = "agentless")] + Agentless(agentless::NativeAgentlessFetcher), +} + pub struct ConfigFetcher { pub file_storage: S, state: Arc>, + mode: FetcherMode, } pub struct ConfigClientState { @@ -215,6 +264,8 @@ pub struct ConfigClientState { /// Services discovered at runtime. Sent to the agent on each poll so it can route configs /// targeting those services to this client. Updated out-of-band by the consumer extra_services: Vec, + /// Server-recommended interval between consecutive polls. + refresh_interval: Option, } impl Default for ConfigClientState { @@ -226,6 +277,7 @@ impl Default for ConfigClientState { root_version: 1, last_error: None, extra_services: vec![], + refresh_interval: None, } } } @@ -234,14 +286,39 @@ impl ConfigClientState { pub fn set_extra_services(&mut self, services: Vec) { self.extra_services = services; } + + pub fn server_recommended_refresh_interval(&self) -> Option { + self.refresh_interval + } } impl ConfigFetcher { - pub fn new(file_storage: S, state: Arc>) -> Self { - ConfigFetcher { + /// Create a new config fetcher + /// This is guaranteed to be immediate (no await point) if `state.invariants.agentless_enabled` + /// is false + pub async fn new( + file_storage: S, + state: Arc>, + ) -> anyhow::Result { + #[cfg(feature = "agentless")] + let mode: FetcherMode = match &state.invariants.agentless { + Some(agentless_cfg) => FetcherMode::Agentless( + agentless::NativeAgentlessFetcher::new( + agentless_cfg.clone(), + state.endpoint.clone(), + ) + .await?, + ), + None => FetcherMode::Agent, + }; + #[cfg(not(feature = "agentless"))] + let mode: FetcherMode = FetcherMode::Agent; + + Ok(ConfigFetcher { file_storage, state, - } + mode, + }) } /// Sets the apply state on a stored file. @@ -324,34 +401,13 @@ impl ConfigFetcher { } } - /// Quite generic fetching implementation: - /// - runs a request against the Remote Config Server, - /// - validates the data, - /// - removes unused files - /// - checks if the files are already known, - /// - stores new files, - /// - returns all currently active files. - /// - /// It also makes sure that old files are dropped before new files are inserted. - /// - /// Returns None if nothing changed. Otherwise Some(active configs). - pub async fn fetch_once( + async fn fetch_agent( &mut self, - runtime_id: &str, + config_req: ClientGetConfigsRequest, target: &Target, - product_capabilities: &ConfigProductCapabilities, - client_id: &str, client_state: &mut ConfigClientState, ) -> anyhow::Result>>> { - let config_req = self.build_config_request( - runtime_id, - target, - product_capabilities, - client_id, - &*client_state, - ); trace!("Submitting remote config request: {config_req:?}"); - let req = self .state .endpoint @@ -557,9 +613,83 @@ impl ConfigFetcher { client_state.last_config_paths = config_paths; Ok(Some(configs)) } + + /// Quite generic fetching implementation: + /// - runs a request against the Remote Config Server, + /// - validates the data, + /// - removes unused files + /// - checks if the files are already known, + /// - stores new files, + /// - returns all currently active files. + /// + /// It also makes sure that old files are dropped before new files are inserted. + /// + /// Returns None if nothing changed. Otherwise Some(active configs). + pub async fn fetch_once( + &mut self, + runtime_id: &str, + target: &Target, + product_capabilities: &ConfigProductCapabilities, + client_id: &str, + client_state: &mut ConfigClientState, + ) -> anyhow::Result>>> { + let config_req = self.build_config_request( + runtime_id, + target, + product_capabilities, + client_id, + &*client_state, + ); + match &mut self.mode { + FetcherMode::Agent => self.fetch_agent(config_req, target, client_state).await, + #[cfg(feature = "agentless")] + FetcherMode::Agentless(agentless_fetcher) => { + #[allow(clippy::expect_used)] + let client = config_req.client.expect( + "RC ConfigFetcher::build_config_request should always return a `Some` client", + ); + + let cache = agentless::TargetCache::new(&self.state, &self.file_storage); + let res = match agentless_fetcher.fetch_config(client, &cache).await { + Ok(r) => r, + Err(e) => { + client_state.last_error = Some(format!("{e:#}")); + // Surface the recommended backoff to the consumer of + // `ConfigClientState::server_recommended_refresh_interval` + // so it waits before the next attempt. `None` means + // "no extra backoff, use the regular interval". + if let Some(backoff) = agentless_fetcher.next_backoff() { + client_state.refresh_interval = Some(backoff); + } + return Err(e); + } + }; + + client_state.root_version = res.root_version; + client_state.targets_version = res.target_version; + client_state.refresh_interval = Some(res.refresh_interval); + if res.opaque_backend_state != client_state.opaque_backend_state { + client_state.opaque_backend_state = res.opaque_backend_state.clone(); + } + client_state.last_error = None; + + let mut config_paths: HashSet = HashSet::new(); + for target_ref in &res.targets { + if let Ok(parsed) = RemoteConfigPath::try_parse(&target_ref.path) { + config_paths.insert(parsed.into()); + } + } + + let configs = cache.collect_handles(&res.targets)?; + + client_state.last_config_paths = config_paths; + Ok(Some(configs)) + } + } + } } -fn get_agent_configs_endpoint(endpoint: &Endpoint) -> Endpoint { +fn make_agent_configs_endpoint(endpoint: &Endpoint) -> Endpoint { let mut parts = endpoint.url.clone().into_parts(); parts.path_and_query = Some(PathAndQuery::from_static("/v0.7/config")); #[allow(clippy::unwrap_used)] @@ -691,7 +821,9 @@ pub mod tests { let mut fetcher = ConfigFetcher::new( storage.clone(), Arc::new(ConfigFetcherState::new(server.dummy_options().invariants)), - ); + ) + .await + .unwrap(); let mut opaque_state = ConfigClientState::default(); let mut response = http_common::empty_response(Response::builder()).unwrap(); @@ -729,6 +861,7 @@ pub mod tests { language: "php".to_string(), tracer_version: "1.2.3".to_string(), endpoint: server.endpoint.clone(), + agentless: None, }; let product_capabilities = ConfigProductCapabilities::new( vec![ @@ -741,7 +874,9 @@ pub mod tests { let mut fetcher = ConfigFetcher::new( storage.clone(), Arc::new(ConfigFetcherState::new(invariants)), - ); + ) + .await + .unwrap(); let mut opaque_state = ConfigClientState::default(); { @@ -925,7 +1060,9 @@ pub mod tests { let mut fetcher = ConfigFetcher::new( storage, Arc::new(ConfigFetcherState::new(server.dummy_options().invariants)), - ); + ) + .await + .unwrap(); let mut opaque_state = ConfigClientState::default(); // Default: nothing set, agent receives an empty list. @@ -1023,7 +1160,9 @@ pub mod tests { let mut fetcher = ConfigFetcher::new( storage, Arc::new(ConfigFetcherState::new(server.dummy_options().invariants)), - ); + ) + .await + .unwrap(); let mut opaque_state = ConfigClientState::default(); let fetched = fetcher diff --git a/libdd-remote-config/src/fetch/mod.rs b/libdd-remote-config/src/fetch/mod.rs index a143195871..43a2ea0603 100644 --- a/libdd-remote-config/src/fetch/mod.rs +++ b/libdd-remote-config/src/fetch/mod.rs @@ -1,13 +1,20 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#[cfg(feature = "agentless")] +mod agentless; + mod fetcher; mod multitarget; mod shared; mod single; + #[cfg(any(test, feature = "test"))] pub mod test_server; +#[cfg(feature = "agentless")] +pub use agentless::*; + #[allow(clippy::useless_attribute)] // different clippy versions are differently picky #[cfg_attr(test, allow(ambiguous_glob_reexports))] // ignore mod tests re-export pub use fetcher::*; diff --git a/libdd-remote-config/src/fetch/shared.rs b/libdd-remote-config/src/fetch/shared.rs index d4e3b6e11f..bb73d7421a 100644 --- a/libdd-remote-config/src/fetch/shared.rs +++ b/libdd-remote-config/src/fetch/shared.rs @@ -275,7 +275,13 @@ impl SharedFetcher { S::StoredFile: RefcountedFile, { let state = storage.state.clone(); - let mut fetcher = ConfigFetcher::new(storage, state); + let mut fetcher = match ConfigFetcher::new(storage, state).await { + Ok(f) => f, + Err(e) => { + error!("failed to create the fetcher: {:?}", e); + return; + } + }; let mut opaque_state = ConfigClientState::default(); @@ -314,7 +320,9 @@ impl SharedFetcher { }; match fetched { - Ok(None) => clean_inactive(), // nothing changed + Ok(None) => { + clean_inactive(); + } Ok(Some(files)) => { if !files.is_empty() || !last_files.is_empty() { for file in files.iter() { @@ -349,6 +357,12 @@ impl SharedFetcher { } } + if let Some(interval) = opaque_state.server_recommended_refresh_interval() { + // Keep the run-loop interval in sync with the server-provided value + self.interval + .store(interval.as_nanos() as u64, Ordering::Relaxed); + } + select! { _ = self.cancellation.cancelled() => { break; } _ = sleep(Duration::from_nanos(self.interval.load(Ordering::Relaxed))) => {} diff --git a/libdd-remote-config/src/fetch/single.rs b/libdd-remote-config/src/fetch/single.rs index bcc4398066..fd340e0a4e 100644 --- a/libdd-remote-config/src/fetch/single.rs +++ b/libdd-remote-config/src/fetch/single.rs @@ -8,6 +8,9 @@ use crate::fetch::{ use crate::file_change_tracker::{Change, ChangeTracker, FilePath, UpdatedFiles}; use crate::{RemoteConfigCapabilities, RemoteConfigPath, RemoteConfigProduct, Target}; use std::sync::Arc; +use std::time::Duration; + +const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(5); /// Simple implementation pub struct SingleFetcher { @@ -16,7 +19,7 @@ pub struct SingleFetcher { product_capabilities: ConfigProductCapabilities, runtime_id: String, client_id: String, - opaque_state: ConfigClientState, + client_state: ConfigClientState, } #[derive(Clone, Debug)] @@ -27,12 +30,40 @@ pub struct ConfigOptions { } impl SingleFetcher { - pub fn new(sink: S, target: Target, runtime_id: String, options: ConfigOptions) -> Self { - SingleFetcher { + pub async fn new( + sink: S, + target: Target, + runtime_id: String, + options: ConfigOptions, + ) -> anyhow::Result { + Ok(SingleFetcher { fetcher: ConfigFetcher::new( sink, Arc::new(ConfigFetcherState::new(options.invariants)), + ) + .await?, + target: Arc::new(target), + product_capabilities: ConfigProductCapabilities::new( + options.products, + options.capabilities, ), + runtime_id, + client_id: uuid::Uuid::new_v4().to_string(), + client_state: ConfigClientState::default(), + }) + } + + pub fn new_no_agentless( + sink: S, + target: Target, + runtime_id: String, + options: ConfigOptions, + ) -> anyhow::Result { + Ok(SingleFetcher { + fetcher: futures::executor::block_on(ConfigFetcher::new( + sink, + Arc::new(ConfigFetcherState::new(options.invariants)), + ))?, target: Arc::new(target), product_capabilities: ConfigProductCapabilities::new( options.products, @@ -40,8 +71,8 @@ impl SingleFetcher { ), runtime_id, client_id: uuid::Uuid::new_v4().to_string(), - opaque_state: ConfigClientState::default(), - } + client_state: ConfigClientState::default(), + }) } pub fn with_client_id(mut self, client_id: String) -> Self { @@ -57,15 +88,24 @@ impl SingleFetcher { &self.target, &self.product_capabilities, self.client_id.as_str(), - &mut self.opaque_state, + &mut self.client_state, ) .await } - pub fn get_client_id(&self) -> &String { + pub fn get_client_id(&self) -> &str { &self.client_id } + /// Returns the server-recommended interval before the next poll. + /// In agentless mode this is updated after every successful fetch. + /// In agent mode it returns the default of 5 seconds. + pub fn get_refresh_interval(&self) -> Duration { + self.client_state + .server_recommended_refresh_interval() + .unwrap_or(DEFAULT_REFRESH_INTERVAL) + } + /// Accesses the underlying file storage (the [`ConfigFetcher`]'s `file_storage`). pub fn file_storage(&self) -> &S { &self.fetcher.file_storage @@ -80,7 +120,7 @@ impl SingleFetcher { /// Sent to the agent on each subsequent poll so it can route configs targeting those /// services to this client. Replace-semantics: the new vec fully overrides the previous one. pub fn set_extra_services(&mut self, services: Vec) { - self.opaque_state.set_extra_services(services); + self.client_state.set_extra_services(services); } /// Replace the set of subscribed products and capabilities. @@ -108,11 +148,28 @@ impl SingleChangesFetcher where S::StoredFile: FilePath, { - pub fn new(sink: S, target: Target, runtime_id: String, options: ConfigOptions) -> Self { - SingleChangesFetcher { + pub async fn new( + sink: S, + target: Target, + runtime_id: String, + options: ConfigOptions, + ) -> anyhow::Result { + Ok(SingleChangesFetcher { changes: ChangeTracker::default(), - fetcher: SingleFetcher::new(sink, target, runtime_id, options), - } + fetcher: SingleFetcher::new(sink, target, runtime_id, options).await?, + }) + } + + pub fn new_no_agentless( + sink: S, + target: Target, + runtime_id: String, + options: ConfigOptions, + ) -> anyhow::Result { + Ok(SingleChangesFetcher { + changes: ChangeTracker::default(), + fetcher: SingleFetcher::new_no_agentless(sink, target, runtime_id, options)?, + }) } pub fn with_client_id(mut self, client_id: String) -> Self { @@ -133,10 +190,16 @@ where }) } - pub fn get_client_id(&self) -> &String { + pub fn get_client_id(&self) -> &str { self.fetcher.get_client_id() } + /// Returns the interval before the next poll. + /// See [`SingleFetcher::get_refresh_interval`]. + pub fn get_refresh_interval(&self) -> Duration { + self.fetcher.get_refresh_interval() + } + /// Sets the apply state on a stored file. pub fn set_config_state(&self, file: &S::StoredFile, state: ConfigApplyState) { self.fetcher.set_config_state(file.path(), state) diff --git a/libdd-remote-config/src/fetch/test_server.rs b/libdd-remote-config/src/fetch/test_server.rs index 3c9c7fc9ec..cc2d5e751c 100644 --- a/libdd-remote-config/src/fetch/test_server.rs +++ b/libdd-remote-config/src/fetch/test_server.rs @@ -216,6 +216,7 @@ impl RemoteConfigServer { language: "php".to_string(), tracer_version: "1.2.3".to_string(), endpoint: self.endpoint.clone(), + agentless: None, }, products: vec![ RemoteConfigProduct::ApmTracing, diff --git a/libdd-remote-config/src/path.rs b/libdd-remote-config/src/path.rs index 5e9f85d066..2972897a22 100644 --- a/libdd-remote-config/src/path.rs +++ b/libdd-remote-config/src/path.rs @@ -81,7 +81,13 @@ impl RemoteConfigPath { if parts.len() != 5 { anyhow::bail!("{} is datadog and does not have exactly 5 parts", path); } - RemoteConfigSource::Datadog(parts[1].parse()?) + let org_id: u64 = parts[1].parse()?; + // The agent parses org_id as an int64; reject values it would + // reject (> i64::MAX) so both clients accept/reject the same paths. + if org_id > i64::MAX as u64 { + anyhow::bail!("org_id {} exceeds i64::MAX in path {}", org_id, path); + } + RemoteConfigSource::Datadog(org_id) } "employee" => { if parts.len() != 4 { @@ -104,8 +110,20 @@ impl RemoteConfigPath { "LIVE_DEBUGGING_SYMBOL_DB" => RemoteConfigProduct::LiveDebuggerSymbolDb, product => anyhow::bail!("Unknown product {}", product), }, - config_id: parts[parts.len() - 2], - name: parts[parts.len() - 1], + config_id: { + let config_id = parts[parts.len() - 2]; + if config_id.is_empty() { + anyhow::bail!("empty config_id in path {}", path); + } + config_id + }, + name: { + let name = parts[parts.len() - 1]; + if name.is_empty() { + anyhow::bail!("empty name in path {}", path); + } + name + }, }) } } diff --git a/libdd-tracer-flare/Cargo.toml b/libdd-tracer-flare/Cargo.toml index 61044913b7..947f5e12da 100644 --- a/libdd-tracer-flare/Cargo.toml +++ b/libdd-tracer-flare/Cargo.toml @@ -35,4 +35,5 @@ tokio = { version = "1.36.0", features = ["time", "macros", "rt"] } [features] default = ["listener"] +agentless = ["libdd-remote-config/agentless"] listener = ["libdd-remote-config/client"] diff --git a/libdd-tracer-flare/src/lib.rs b/libdd-tracer-flare/src/lib.rs index e093dd8cb9..b495cdf0f7 100644 --- a/libdd-tracer-flare/src/lib.rs +++ b/libdd-tracer-flare/src/lib.rs @@ -181,6 +181,7 @@ impl TracerFlareManager { language, tracer_version, endpoint: remote_config_endpoint, + agentless: None, }, products: vec![ RemoteConfigProduct::AgentConfig, @@ -189,12 +190,15 @@ impl TracerFlareManager { capabilities: vec![], }; - tracer_flare.listener = Some(SingleChangesFetcher::new( - ParsedFileStorage::default(), - Target::new(service, env, app_version, vec![], vec![]), - runtime_id, - config_to_fetch, - )); + tracer_flare.listener = Some( + SingleChangesFetcher::new_no_agentless( + ParsedFileStorage::default(), + Target::new(service, env, app_version, vec![], vec![]), + runtime_id, + config_to_fetch, + ) + .map_err(|e| FlareError::ListeningError(e.to_string()))?, + ); Ok(tracer_flare) }