feat(metrics-v3): add Datadog V3 payload encoder

Allows to encode metric payloads using V3 columnar format.

This library enables users to encode their metric payloads using an
efficient column-based protocol. For performance and compatibility
reasons (we also want to keep this crate `no_std` so we could use it in
very resource-constrained environments) this crate does manual protobuf
serialization.

Signed-off-by: Mark Kirichenko <mark.kirichenko@datadoghq.com>

Author: atanzu

.github/CODEOWNERS

@@ -55,6 +55,7 @@ libdd-http-client                    @DataDog/apm-common-components-core
 libdd-agent-client                   @DataDog/apm-common-components-core
 libdd-library-config*/               @DataDog/apm-sdk-capabilities-rust
 libdd-log*/                          @DataDog/apm-common-components-core
+libdd-metrics-v3/                    @DataDog/libdatadog @DataDog/agent-metric-pipelines
 libdd-otel-thread-ctx/               @DataDog/apm-common-components-core
 libdd-otel-thread-ctx-ffi/           @DataDog/apm-common-components-core
 libdd-profiling*/                    @DataDog/libdatadog-profiling

.github/workflows/diff-agent-payload-proto.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/
+# SPDX-License-Identifier: Apache-2.0
+
+set -eux
+
+PROTO_FILE=""
+AGENT_PAYLOAD_COMMIT=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --file)
+      PROTO_FILE=$2
+      shift && shift # past argument and value
+      ;;
+    --commit)
+      AGENT_PAYLOAD_COMMIT=$2
+      shift && shift # past argument and value
+      ;;
+    *)
+      echo "Unknown option $1"
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "$PROTO_FILE" ]; then
+  echo "Missing --file argument"
+  exit 1
+fi
+
+if [ -z "$AGENT_PAYLOAD_COMMIT" ]; then
+  echo "Missing --commit argument"
+  exit 1
+fi
+
+# Vendored files must stay byte-for-byte identical to their pinned commit in agent-payload from
+# `syntax = ...;` onward, so unlike diff-proto-files.sh (which fixes up import/package names for
+# datadog-agent's proto layout) this only strips the local "Vendored from:" preamble comment
+# before diffing, with no other rewriting.
+curl -sf "https://raw.githubusercontent.com/DataDog/agent-payload/$AGENT_PAYLOAD_COMMIT/proto/metrics/$PROTO_FILE" |
+  diff -u <(sed -n '/^syntax = /,$p' "$PROTO_FILE") -

.github/workflows/no-std-metrics-v3.yml

@@ -0,0 +1,40 @@
+name: 'No-std check (libdd-metrics-v3)'
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - mq-working-branch-*
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+jobs:
+  no-std-check:
+    # `#![no_std]` alone only stops this crate's own code from referencing `std`; a dependency
+    # that quietly requires std would still link fine on a normal host target. Building for a
+    # target with no OS (so no std is even available to link against) is what actually proves the
+    # whole dependency graph is no_std-compatible.
+    name: "libdd-metrics-v3 builds for a target with no OS and no std"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+      - name: Read Rust version from rust-toolchain.toml
+        id: rust-version
+        run: echo "version=$(grep -Po '^channel = "\K[^"]+' rust-toolchain.toml)" >> $GITHUB_OUTPUT
+      - name: Install ${{ steps.rust-version.outputs.version }} toolchain
+        run: |
+          rustup set profile minimal
+          rustup install ${{ steps.rust-version.outputs.version }}
+          rustup default ${{ steps.rust-version.outputs.version }}
+          rustup target add x86_64-unknown-none
+          rustup component add clippy
+      - name: Cache [rust]
+        uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # 2.8.1
+        with:
+          cache-targets: true # cache build artifacts
+          cache-bin: true # cache the ~/.cargo/bin directory
+      - name: Build for x86_64-unknown-none
+        run: cargo build -p libdd-metrics-v3 --target x86_64-unknown-none
+      - name: Clippy for x86_64-unknown-none
+        run: cargo clippy -p libdd-metrics-v3 --target x86_64-unknown-none -- -D warnings

.github/workflows/verify-metrics-v3-proto.yml

@@ -0,0 +1,37 @@
+name: 'Verify metrics-v3 proto'
+on:
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+env:
+  # Pinned commit of https://github.com/DataDog/agent-payload/blob/master/proto/metrics/intake_v3.proto
+  AGENT_PAYLOAD_COMMIT: "512d523e386f75077efe9e8d878284104d491f46"
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+jobs:
+  verify-proto-files:
+    name: "Verify libdd-metrics-v3's vendored intake_v3.proto and generated bindings are in sync"
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+      - name: diff intake_v3.proto against agent-payload
+        working-directory: libdd-metrics-v3/proto
+        run: |
+          ../../.github/workflows/diff-agent-payload-proto.sh --file intake_v3.proto --commit "$AGENT_PAYLOAD_COMMIT"
+      - name: Read Rust version from rust-toolchain.toml
+        id: rust-version
+        run: echo "version=$(grep -Po '^channel = "\K[^"]+' rust-toolchain.toml)" >> $GITHUB_OUTPUT
+      - name: Install ${{ steps.rust-version.outputs.version }} toolchain
+        run: rustup set profile minimal && rustup install ${{ steps.rust-version.outputs.version }} && rustup default ${{ steps.rust-version.outputs.version }}
+      - name: Cache [rust]
+        uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # 2.8.1
+        with:
+          cache-targets: true # cache build artifacts
+          cache-bin: true # cache the ~/.cargo/bin directory
+      - name: diff tests/pb/mod.rs
+        working-directory: libdd-metrics-v3
+        run: |
+          cargo build --features generate-protobuf
+          git diff --exit-code -- tests/pb/mod.rs

Cargo.lock

@@ -61,6 +61,7 @@ members = [
   "libdd-log",
   "libdd-log-ffi",
   "libdd-sampling",
+  "libdd-metrics-v3",
 ]
 
 # https://doc.rust-lang.org/cargo/reference/resolver.html

Cargo.toml

@@ -0,0 +1,42 @@
+# Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "libdd-metrics-v3"
+version = "0.1.0"
+description = "V3 columnar protobuf codec for Datadog metrics"
+homepage = "https://github.com/DataDog/libdatadog/tree/main/libdd-metrics-v3"
+repository = "https://github.com/DataDog/libdatadog/tree/main/libdd-metrics-v3"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+autobenches = false
+
+[lints]
+workspace = true
+
+[lib]
+bench = false
+
+[dependencies]
+# default-features = false: enabling foldhash's default "std" feature would pull `std` back into
+# this otherwise `no_std` crate.
+foldhash = { version = "0.2", default-features = false }
+hashbrown = { version = "0.16", default-features = false }
+
+[dev-dependencies]
+# Reference implementation used by the wire-format parity tests in `tests/`: encodes the same
+# columnar data via prost's generated bindings for `proto/intake_v3.proto` and checks that it's
+# byte-for-byte identical to this crate's hand-rolled encoder output.
+bolero = "0.13"
+prost = "0.14"
+
+[build-dependencies]
+prost-build = { version = "0.14", optional = true }
+protoc-bin-vendored = { version = "3.0", optional = true }
+
+[features]
+# Regenerates `tests/pb/mod.rs` from `proto/intake_v3.proto`. Not needed for normal builds or test
+# runs: the generated bindings are checked into version control, and CI verifies they're in sync
+# with the vendored .proto file (see `.github/workflows/verify-metrics-v3-proto.yml`).
+generate-protobuf = ["dep:prost-build", "dep:protoc-bin-vendored"]

libdd-metrics-v3/Cargo.toml

@@ -0,0 +1,76 @@
+// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+// This build script always runs on the host with std available (the `no_std` library it builds
+// has nothing to do with the build script's own environment), so the workspace-wide
+// std-vs-core/alloc and expect-used lints (meant to guard the library itself) don't apply here.
+#![allow(
+    clippy::std_instead_of_core,
+    clippy::std_instead_of_alloc,
+    clippy::expect_used
+)]
+
+fn main() {
+    #[cfg(feature = "generate-protobuf")]
+    generate::run().expect("failed to regenerate protobuf bindings from proto/intake_v3.proto");
+    #[cfg(not(feature = "generate-protobuf"))]
+    println!("cargo:rerun-if-changed=build.rs");
+}
+
+/// Regenerates `tests/pb/mod.rs` from `proto/intake_v3.proto`. Only compiled when the
+/// `generate-protobuf` feature is enabled, since it's the only fallible part of this build
+/// script: normal builds and test runs use the checked-in `tests/pb/mod.rs` and never reach it.
+#[cfg(feature = "generate-protobuf")]
+mod generate {
+    use std::error::Error;
+    use std::{env, fs, path::Path};
+
+    const HEADER: &str = "// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+// This file is @generated by prost-build from `proto/intake_v3.proto`. Do not edit it directly:
+// regenerate it with `cargo build -p libdd-metrics-v3 --features generate-protobuf` after
+// changing the vendored .proto file.
+#![allow(dead_code, clippy::all, clippy::pedantic, clippy::nursery)]
+
+";
+
+    pub fn run() -> Result<(), Box<dyn Error>> {
+        // protoc is required to compile proto files. This uses protoc-bin-vendored to obtain a
+        // protoc binary, setting the env var to tell prost-build where to find it.
+        env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?);
+
+        let cur_working_dir = env::var("CARGO_MANIFEST_DIR")?;
+        let out_dir = Path::new(&cur_working_dir).join("tests/pb");
+        fs::create_dir_all(&out_dir)?;
+
+        let mut config = prost_build::Config::new();
+        config.out_dir(&out_dir);
+
+        println!("cargo:rerun-if-changed=proto/intake_v3.proto");
+        config.compile_protos(&["proto/intake_v3.proto"], &["proto"])?;
+
+        // prost-build names the output file after the proto's package
+        // (`datadoghq.api.metrics.v3.rs`); rename it to `mod.rs` so `tests/parity.rs` can
+        // pull it in as a plain submodule.
+        let generated = out_dir.join("datadoghq.api.metrics.v3.rs");
+        let mod_rs = out_dir.join("mod.rs");
+        fs::rename(&generated, &mod_rs)?;
+
+        prepend_to_file(HEADER.as_bytes(), &mod_rs)?;
+
+        Ok(())
+    }
+
+    fn prepend_to_file(data: &[u8], file_path: &Path) -> Result<(), Box<dyn Error>> {
+        use std::io::{Read, Write};
+
+        let mut f = fs::File::open(file_path)?;
+        let mut content = data.to_owned();
+        f.read_to_end(&mut content)?;
+
+        let mut f = fs::File::create(file_path)?;
+        f.write_all(content.as_slice())?;
+        Ok(())
+    }
+}

libdd-metrics-v3/build.rs

@@ -0,0 +1,90 @@
+// Vendored from:
+// https://github.com/DataDog/agent-payload/blob/512d523e386f75077efe9e8d878284104d491f46/proto/metrics/intake_v3.proto
+//
+// This is the wire format that `libdd-metrics-v3`'s hand-rolled encoder (see `src/writer.rs`)
+// implements without a Protocol Buffers library. `.github/workflows/verify-metrics-v3-proto.yml`
+// checks in CI that the content below stays byte-for-byte identical to the pinned commit above,
+// and `tests/parity.rs` checks that our hand-rolled encoder's output is byte-identical to what
+// prost's generated bindings for this file produce from the same data.
+
+syntax = "proto3";
+
+package datadoghq.api.metrics.v3;
+
+option go_package = "github.com/DataDog/agent-payload/v5/metrics/intake_v3";
+
+message Payload {
+  reserved 1; // for compatibility with agentpayload.MetricPayload.series
+  Metadata metadata = 2;
+  MetricData metricData = 3;
+}
+
+message Metadata {
+  repeated string tags = 1;
+  repeated string resources = 2; // even number of elements, [Type, Name] pairs
+}
+
+message MetricData {
+  // Dictionaries
+  // All dictionary indexes are base-1, zero implicitly represents an empty value.
+  bytes dictNameStr = 1;                // varint length + value
+  bytes dictTagStr = 2;                 // varint length + value
+  repeated sint64 dictTagsets = 3;      // length, delta encoded set of indexes into dictTagsStr 
+
+  bytes dictResourceStr = 4;            // varint length + value
+  repeated int64 dictResourceLen = 5;   // number of elements in Type and Name arrays
+  repeated sint64 dictResourceType = 6; // delta encoded set of indexes into dictResourceStr
+  repeated sint64 dictResourceName = 7; // delta encoded set of indexes into dictResourceStr
+
+  bytes dictSourceTypeName = 8;         // varint length + value
+  repeated int32 dictOriginInfo = 9;    // (product, category, service) tuples
+  bytes dictUnitStr = 25;                // varint length + value
+
+  // One entry per time series
+  repeated uint64 types = 10;           // type = metricType | valueType | metricFlags
+  repeated sint64 nameRefs = 11;        // index into dictNameStr, entire array is delta encoded
+  repeated sint64 tagsetRefs = 12;      // index into dictTagsets, entire array is delta encoded
+  repeated sint64 resourcesRefs = 13;   // index into dictResourceLen, entire array is delta encoded
+  repeated uint64 intervals = 14;
+  repeated uint64 numPoints = 15;
+  repeated sint64 sourceTypeNameRefs = 23;  // index into dictSourceTypeName, entire array is delta encoded
+  repeated sint64 originInfoRefs = 24;      // index into dictOriginInfo, entire array is delta encoded
+  repeated sint64 unitRefs = 26;            // index into dictUnitStr, value present if flagHasUnit is set, entire array is delta encoded
+
+  // each metric has numPoints values in this section
+  repeated sint64 timestamps = 16;      // entire array delta encoded
+  repeated sint64 valsSint64 = 17;      // or
+  repeated float valsFloat32 = 18;      // or
+  repeated double valsFloat64 = 19;     // based on valueType
+  repeated uint64 sketchNumBins = 20;
+  repeated sint32 sketchBinKeys = 21;   // per-metric sequence is delta encoded
+  repeated uint32 sketchBinCnts = 22;
+  // sketch summary Sum, Min, Max are encoded as three consecutive elements in one of vals using valueType
+  // sketch summary Cnt is always encoded in valInt64
+  // sketch summary Avg is reconstructed as Sum/Cnt in the intake
+}
+
+enum metricType {
+  UNUSED = 0;
+  Count = 1;
+  Rate = 2;
+  Gauge = 3;
+  Sketch = 4;
+}
+
+enum valueType {
+  Zero    = 0x00;  // value is zero, not stored explicitly
+  Sint64  = 0x10;  // value is stored in valsSint64
+  Float32 = 0x20;  // value is stored in valsFloat32
+  Float64 = 0x30;  // value is stored in valsFloat64
+}
+
+enum metricFlags {
+  flagNone    = 0;
+  flagNoIndex = 0x100; // metric should not be indexed (equivalent to origin metric type == agent_hidden in v2)
+  flagHasUnit = 0x200; // timeseries has a unit in the unitRefs column
+}
+
+message Response {
+  string error = 1;
+}