From 2e9f37834d075db71e517f29d980bad8fc2aa389 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 15 May 2026 15:12:45 -0400 Subject: [PATCH 01/43] chore(buffers): replace string metric prefixes with typed BufferChannelKind enum --- .../src/topology/channel/limited_queue.rs | 82 +++++++++++++------ .../src/topology/channel/mod.rs | 4 +- .../src/internal_event/metric_name.rs | 28 +++++++ lib/vector-core/src/source_sender/output.rs | 5 +- src/topology/builder.rs | 5 +- 5 files changed, 89 insertions(+), 35 deletions(-) diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index 3e1f3f7c98c93..68ca0954bfaef 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -16,14 +16,23 @@ use std::sync::Mutex; use async_stream::stream; use crossbeam_queue::{ArrayQueue, SegQueue}; use futures::Stream; -use metrics::{Gauge, Histogram, gauge, histogram}; +use metrics::{Gauge, Histogram}; use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; +use vector_common::internal_event::{GaugeName, HistogramName}; use vector_common::stats::TimeEwmaGauge; +use vector_common::{gauge, histogram}; use crate::{InMemoryBufferable, config::MemoryBufferSize}; pub const DEFAULT_EWMA_HALF_LIFE_SECONDS: f64 = 5.0; +/// Identifies which buffer channel is being instrumented, determining the metric name prefix. +#[derive(Clone, Copy, Debug)] +pub enum BufferChannelKind { + Source, + Transform, +} + /// Error returned by `LimitedSender::send` when the receiver has disconnected. #[derive(Debug, PartialEq, Eq)] pub struct SendError(pub T); @@ -99,13 +108,13 @@ where #[derive(Clone, Debug)] pub struct ChannelMetricMetadata { - prefix: &'static str, + kind: BufferChannelKind, output: Option, } impl ChannelMetricMetadata { - pub fn new(prefix: &'static str, output: Option) -> Self { - Self { prefix, output } + pub fn new(kind: BufferChannelKind, output: Option) -> Self { + Self { kind, output } } } @@ -127,7 +136,6 @@ struct Metrics { impl Metrics { #[expect(clippy::cast_precision_loss)] // We have to convert buffer sizes for a gauge, it's okay to lose precision here. - #[allow(clippy::disallowed_macros)] // Metric names are constructed dynamically from runtime prefixes. fn new( limit: MemoryBufferSize, metadata: ChannelMetricMetadata, @@ -135,34 +143,54 @@ impl Metrics { ) -> Self { let ewma_half_life_seconds = ewma_half_life_seconds.unwrap_or(DEFAULT_EWMA_HALF_LIFE_SECONDS); - let ChannelMetricMetadata { prefix, output } = metadata; - let (legacy_suffix, gauge_suffix, max_value) = match limit { - MemoryBufferSize::MaxEvents(max_events) => ( - "_max_event_size", - "_max_size_events", - max_events.get() as f64, - ), - MemoryBufferSize::MaxSize(max_bytes) => { - ("_max_byte_size", "_max_size_bytes", max_bytes.get() as f64) - } - }; - let max_gauge_name = format!("{prefix}{gauge_suffix}"); - let legacy_max_gauge_name = format!("{prefix}{legacy_suffix}"); - let histogram_name = format!("{prefix}_utilization"); - let gauge_name = format!("{prefix}_utilization_level"); - let mean_name = format!("{prefix}_utilization_mean"); + let ChannelMetricMetadata { kind, output } = metadata; + let (max_name, legacy_name, histogram_name, level_name, mean_name, max_value) = + match (kind, limit) { + (BufferChannelKind::Source, MemoryBufferSize::MaxEvents(n)) => ( + GaugeName::SourceBufferMaxSizeEvents, + GaugeName::SourceBufferMaxEventSize, + HistogramName::SourceBufferUtilization, + GaugeName::SourceBufferUtilizationLevel, + GaugeName::SourceBufferUtilizationMean, + n.get() as f64, + ), + (BufferChannelKind::Source, MemoryBufferSize::MaxSize(n)) => ( + GaugeName::SourceBufferMaxSizeBytes, + GaugeName::SourceBufferMaxByteSize, + HistogramName::SourceBufferUtilization, + GaugeName::SourceBufferUtilizationLevel, + GaugeName::SourceBufferUtilizationMean, + n.get() as f64, + ), + (BufferChannelKind::Transform, MemoryBufferSize::MaxEvents(n)) => ( + GaugeName::TransformBufferMaxSizeEvents, + GaugeName::TransformBufferMaxEventSize, + HistogramName::TransformBufferUtilization, + GaugeName::TransformBufferUtilizationLevel, + GaugeName::TransformBufferUtilizationMean, + n.get() as f64, + ), + (BufferChannelKind::Transform, MemoryBufferSize::MaxSize(n)) => ( + GaugeName::TransformBufferMaxSizeBytes, + GaugeName::TransformBufferMaxByteSize, + HistogramName::TransformBufferUtilization, + GaugeName::TransformBufferUtilizationLevel, + GaugeName::TransformBufferUtilizationMean, + n.get() as f64, + ), + }; #[cfg(test)] let recorded_values = Arc::new(Mutex::new(Vec::new())); if let Some(label_value) = output { - let max_gauge = gauge!(max_gauge_name, "output" => label_value.clone()); + let max_gauge = gauge!(max_name, "output" => label_value.clone()); max_gauge.set(max_value); let mean_gauge_handle = gauge!(mean_name, "output" => label_value.clone()); // DEPRECATED: buffer-bytes-events-metrics - let legacy_max_gauge = gauge!(legacy_max_gauge_name, "output" => label_value.clone()); + let legacy_max_gauge = gauge!(legacy_name, "output" => label_value.clone()); legacy_max_gauge.set(max_value); Self { histogram: histogram!(histogram_name, "output" => label_value.clone()), - gauge: gauge!(gauge_name, "output" => label_value.clone()), + gauge: gauge!(level_name, "output" => label_value.clone()), mean_gauge: TimeEwmaGauge::new(mean_gauge_handle, ewma_half_life_seconds), max_gauge, legacy_max_gauge, @@ -170,15 +198,15 @@ impl Metrics { recorded_values, } } else { - let max_gauge = gauge!(max_gauge_name); + let max_gauge = gauge!(max_name); max_gauge.set(max_value); let mean_gauge_handle = gauge!(mean_name); // DEPRECATED: buffer-bytes-events-metrics - let legacy_max_gauge = gauge!(legacy_max_gauge_name); + let legacy_max_gauge = gauge!(legacy_name); legacy_max_gauge.set(max_value); Self { histogram: histogram!(histogram_name), - gauge: gauge!(gauge_name), + gauge: gauge!(level_name), mean_gauge: TimeEwmaGauge::new(mean_gauge_handle, ewma_half_life_seconds), max_gauge, legacy_max_gauge, diff --git a/lib/vector-buffers/src/topology/channel/mod.rs b/lib/vector-buffers/src/topology/channel/mod.rs index e0180111819cb..ab8e0fa62e5ce 100644 --- a/lib/vector-buffers/src/topology/channel/mod.rs +++ b/lib/vector-buffers/src/topology/channel/mod.rs @@ -3,8 +3,8 @@ mod receiver; mod sender; pub use limited_queue::{ - ChannelMetricMetadata, DEFAULT_EWMA_HALF_LIFE_SECONDS, LimitedReceiver, LimitedSender, - SendError, limited, + BufferChannelKind, ChannelMetricMetadata, DEFAULT_EWMA_HALF_LIFE_SECONDS, LimitedReceiver, + LimitedSender, SendError, limited, }; pub use receiver::*; pub use sender::*; diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index a1438465518e2..f520a3ecee316 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -127,6 +127,8 @@ pub enum HistogramName { HttpClientRttSeconds, HttpClientResponseRttSeconds, HttpClientErrorRttSeconds, + SourceBufferUtilization, + TransformBufferUtilization, } impl HistogramName { @@ -160,6 +162,8 @@ impl HistogramName { Self::HttpClientRttSeconds => "http_client_rtt_seconds", Self::HttpClientResponseRttSeconds => "http_client_response_rtt_seconds", Self::HttpClientErrorRttSeconds => "http_client_error_rtt_seconds", + Self::SourceBufferUtilization => "source_buffer_utilization", + Self::TransformBufferUtilization => "transform_buffer_utilization", } } } @@ -168,6 +172,18 @@ impl HistogramName { #[strum(serialize_all = "snake_case")] pub enum GaugeName { ComponentLatencyMeanSeconds, + SourceBufferMaxSizeEvents, + SourceBufferMaxSizeBytes, + SourceBufferMaxEventSize, + SourceBufferMaxByteSize, + SourceBufferUtilizationLevel, + SourceBufferUtilizationMean, + TransformBufferMaxSizeEvents, + TransformBufferMaxSizeBytes, + TransformBufferMaxEventSize, + TransformBufferMaxByteSize, + TransformBufferUtilizationLevel, + TransformBufferUtilizationMean, BufferMaxSizeEvents, BufferMaxEventSize, BufferMaxSizeBytes, @@ -199,6 +215,18 @@ impl GaugeName { pub const fn as_str(self) -> &'static str { match self { Self::ComponentLatencyMeanSeconds => "component_latency_mean_seconds", + Self::SourceBufferMaxSizeEvents => "source_buffer_max_size_events", + Self::SourceBufferMaxSizeBytes => "source_buffer_max_size_bytes", + Self::SourceBufferMaxEventSize => "source_buffer_max_event_size", + Self::SourceBufferMaxByteSize => "source_buffer_max_byte_size", + Self::SourceBufferUtilizationLevel => "source_buffer_utilization_level", + Self::SourceBufferUtilizationMean => "source_buffer_utilization_mean", + Self::TransformBufferMaxSizeEvents => "transform_buffer_max_size_events", + Self::TransformBufferMaxSizeBytes => "transform_buffer_max_size_bytes", + Self::TransformBufferMaxEventSize => "transform_buffer_max_event_size", + Self::TransformBufferMaxByteSize => "transform_buffer_max_byte_size", + Self::TransformBufferUtilizationLevel => "transform_buffer_utilization_level", + Self::TransformBufferUtilizationMean => "transform_buffer_utilization_mean", Self::BufferMaxSizeEvents => "buffer_max_size_events", Self::BufferMaxEventSize => "buffer_max_event_size", Self::BufferMaxSizeBytes => "buffer_max_size_bytes", diff --git a/lib/vector-core/src/source_sender/output.rs b/lib/vector-core/src/source_sender/output.rs index 4a194cc3e1c1f..4d9e0ef61683c 100644 --- a/lib/vector-core/src/source_sender/output.rs +++ b/lib/vector-core/src/source_sender/output.rs @@ -11,7 +11,7 @@ use metrics::Histogram; use tracing::Span; use vector_buffers::{ config::MemoryBufferSize, - topology::channel::{self, ChannelMetricMetadata, LimitedReceiver, LimitedSender}, + topology::channel::{self, BufferChannelKind, ChannelMetricMetadata, LimitedReceiver, LimitedSender}, }; use vector_common::{ byte_size_of::ByteSizeOf, @@ -30,7 +30,6 @@ use crate::{ schema::Definition, }; -const UTILIZATION_METRIC_PREFIX: &str = "source_buffer"; /// UnsentEvents tracks the number of events yet to be sent in the buffer. This is used to /// increment the appropriate counters when a future is not polled to completion. Particularly, @@ -140,7 +139,7 @@ impl Output { ) -> (Self, LimitedReceiver) { let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(n).unwrap()); let channel_metrics = - ChannelMetricMetadata::new(UTILIZATION_METRIC_PREFIX, Some(output.clone())); + ChannelMetricMetadata::new(BufferChannelKind::Source, Some(output.clone())); let (tx, rx) = channel::limited(limit, Some(channel_metrics), ewma_half_life_seconds); ( Self { diff --git a/src/topology/builder.rs b/src/topology/builder.rs index a25a99d337a26..d0a742db52e59 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -21,7 +21,7 @@ use vector_lib::{ BufferType, WhenFull, topology::{ builder::TopologyBuilder, - channel::{BufferReceiver, BufferSender, ChannelMetricMetadata, LimitedReceiver}, + channel::{BufferChannelKind, BufferReceiver, BufferSender, ChannelMetricMetadata, LimitedReceiver}, }, }, internal_event::{self, CountByteSize, EventsSent, InternalEventHandle as _, Registered}, @@ -67,7 +67,6 @@ pub(crate) static SOURCE_SENDER_BUFFER_SIZE: LazyLock = const READY_ARRAY_CAPACITY: NonZeroUsize = NonZeroUsize::new(CHUNK_SIZE * 4).unwrap(); pub(crate) const TOPOLOGY_BUFFER_SIZE: NonZeroUsize = NonZeroUsize::new(100).unwrap(); -const TRANSFORM_CHANNEL_METRIC_PREFIX: &str = "transform_buffer"; static TRANSFORM_CONCURRENCY_LIMIT: LazyLock = LazyLock::new(|| { crate::app::worker_threads() @@ -508,7 +507,7 @@ impl<'a> Builder<'a> { Ok(transform) => transform, }; - let metrics = ChannelMetricMetadata::new(TRANSFORM_CHANNEL_METRIC_PREFIX, None); + let metrics = ChannelMetricMetadata::new(BufferChannelKind::Transform, None); let (input_tx, input_rx) = TopologyBuilder::standalone_memory( TOPOLOGY_BUFFER_SIZE, WhenFull::Block, From 8ed979c25086a3103d13b67b5712a088211044a2 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 15 May 2026 15:13:46 -0400 Subject: [PATCH 02/43] fmt --- lib/vector-core/src/source_sender/output.rs | 5 +++-- src/topology/builder.rs | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/vector-core/src/source_sender/output.rs b/lib/vector-core/src/source_sender/output.rs index 4d9e0ef61683c..fc596c6b9a50b 100644 --- a/lib/vector-core/src/source_sender/output.rs +++ b/lib/vector-core/src/source_sender/output.rs @@ -11,7 +11,9 @@ use metrics::Histogram; use tracing::Span; use vector_buffers::{ config::MemoryBufferSize, - topology::channel::{self, BufferChannelKind, ChannelMetricMetadata, LimitedReceiver, LimitedSender}, + topology::channel::{ + self, BufferChannelKind, ChannelMetricMetadata, LimitedReceiver, LimitedSender, + }, }; use vector_common::{ byte_size_of::ByteSizeOf, @@ -30,7 +32,6 @@ use crate::{ schema::Definition, }; - /// UnsentEvents tracks the number of events yet to be sent in the buffer. This is used to /// increment the appropriate counters when a future is not polled to completion. Particularly, /// this is known to happen in a Warp server when a client sends a new HTTP request on a TCP diff --git a/src/topology/builder.rs b/src/topology/builder.rs index d0a742db52e59..2d9d9f7d5cf81 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -21,7 +21,10 @@ use vector_lib::{ BufferType, WhenFull, topology::{ builder::TopologyBuilder, - channel::{BufferChannelKind, BufferReceiver, BufferSender, ChannelMetricMetadata, LimitedReceiver}, + channel::{ + BufferChannelKind, BufferReceiver, BufferSender, ChannelMetricMetadata, + LimitedReceiver, + }, }, }, internal_event::{self, CountByteSize, EventsSent, InternalEventHandle as _, Registered}, From 169c4ff4f7881beba0393f5c2a19a8432030c634 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 15 May 2026 15:18:01 -0400 Subject: [PATCH 03/43] fix(buffers): update test usages of ChannelMetricMetadata to use BufferChannelKind --- lib/vector-buffers/src/topology/channel/limited_queue.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index 68ca0954bfaef..3252969bfc292 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -543,7 +543,7 @@ mod tests { let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap()); let (mut tx, mut rx) = limited( limit, - Some(ChannelMetricMetadata::new("test_channel", None)), + Some(ChannelMetricMetadata::new(BufferChannelKind::Source, None)), None, ); @@ -565,7 +565,7 @@ mod tests { let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap()); let (mut tx, mut rx) = limited( limit, - Some(ChannelMetricMetadata::new("test_channel_oversized", None)), + Some(ChannelMetricMetadata::new(BufferChannelKind::Source, None)), None, ); let metrics = tx.inner.metrics.as_ref().unwrap().recorded_values.clone(); @@ -948,7 +948,7 @@ mod tests { let limit = NonZeroUsize::new(size * 10).unwrap(); let (tx, rx) = limited( MemoryBufferSize::MaxEvents(limit), - Some(ChannelMetricMetadata::new("test_channel_concurrent", None)), + Some(ChannelMetricMetadata::new(BufferChannelKind::Source, None)), None, ); let metrics = tx.inner.metrics.as_ref().unwrap().recorded_values.clone(); From 19e994fd315d81d407c4588d1a05c223c4e749fa Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 15 May 2026 15:18:16 -0400 Subject: [PATCH 04/43] fix(buffers): import BufferChannelKind in test module --- lib/vector-buffers/src/topology/channel/limited_queue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index 3252969bfc292..40ebc1ad1a29c 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -502,7 +502,7 @@ mod tests { use tokio_test::{assert_pending, assert_ready, task::spawn}; use vector_common::byte_size_of::ByteSizeOf; - use super::{ChannelMetricMetadata, LimitedReceiver, LimitedSender, limited}; + use super::{BufferChannelKind, ChannelMetricMetadata, LimitedReceiver, LimitedSender, limited}; use crate::{ MemoryBufferSize, test::MultiEventRecord, From 61468e9d2766a657803d09e9fb27150c83be4d38 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 15 May 2026 15:38:39 -0400 Subject: [PATCH 05/43] fmt --- lib/vector-buffers/src/topology/channel/limited_queue.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index 40ebc1ad1a29c..e341c76446cbe 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -502,7 +502,9 @@ mod tests { use tokio_test::{assert_pending, assert_ready, task::spawn}; use vector_common::byte_size_of::ByteSizeOf; - use super::{BufferChannelKind, ChannelMetricMetadata, LimitedReceiver, LimitedSender, limited}; + use super::{ + BufferChannelKind, ChannelMetricMetadata, LimitedReceiver, LimitedSender, limited, + }; use crate::{ MemoryBufferSize, test::MultiEventRecord, From 9446e0335ef5d25b8ef6e387434f4c714e2b2871 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 15 May 2026 16:29:41 -0400 Subject: [PATCH 06/43] chore(metrics): add rustdoc and Configurable derive to metric name enums --- .../src/internal_event/metric_name.rs | 304 +++++++++++++++++- 1 file changed, 303 insertions(+), 1 deletion(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index a1438465518e2..0d90c18664adf 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -1,131 +1,377 @@ use strum::{AsRefStr, Display, EnumIter}; +use vector_config::configurable_component; -/// Canonical list of all per-component internal metric names emitted by Vector. +/// Canonical list of all per-component internal counter metric names emitted by Vector. +#[configurable_component] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] pub enum CounterName { + /// The number of events accepted by this component either from tagged + /// origins like file and uri, or cumulatively from other origins. ComponentReceivedEventsTotal, + + /// The number of event bytes accepted by this component either from + /// tagged origins like file and uri, or cumulatively from other origins. ComponentReceivedEventBytesTotal, + + /// The number of raw bytes accepted by this component from source origins. ComponentReceivedBytesTotal, + + /// The total number of events emitted by this component. ComponentSentEventsTotal, + + /// The total number of event bytes emitted by this component. ComponentSentEventBytesTotal, + + /// The number of raw bytes sent by this component to destination sinks. ComponentSentBytesTotal, + + /// The number of events dropped by this component. ComponentDiscardedEventsTotal, + + /// The total number of errors encountered by this component. ComponentErrorsTotal, + + /// The total number of events for which this source responded with a timeout error. ComponentTimedOutEventsTotal, + + /// The total number of requests for which this source responded with a timeout error. ComponentTimedOutRequestsTotal, + + /// The number of events received by this buffer. BufferReceivedEventsTotal, + + /// The number of bytes received by this buffer. BufferReceivedBytesTotal, + + /// The number of events sent by this buffer. BufferSentEventsTotal, + + /// The number of bytes sent by this buffer. BufferSentBytesTotal, + + /// The number of events dropped by this non-blocking buffer. BufferDiscardedEventsTotal, + + /// The number of bytes dropped by this non-blocking buffer. BufferDiscardedBytesTotal, + + /// The total number of buffer errors encountered. BufferErrorsTotal, + // Internal events from src/internal_events/ + /// The number of events recorded by the aggregate transform. AggregateEventsRecordedTotal, + + /// The number of failed metric updates, `incremental` adds, encountered by the aggregate transform. AggregateFailedUpdates, + + /// The number of flushes done by the aggregate transform. AggregateFlushesTotal, + + /// The number of times the Vector API has been started. ApiStartedTotal, + + /// The total number of files checkpointed. CheckpointsTotal, + + /// The total number of errors identifying files via checksum. ChecksumErrorsTotal, + + /// The total number of metrics collections completed for this component. CollectCompletedTotal, + + /// The total number of times a command has been executed. CommandExecutedTotal, + + /// The total number of times a connection has been established. ConnectionEstablishedTotal, + + /// The total number of errors sending data via the connection. ConnectionSendErrorsTotal, + + /// The total number of times the connection has been shut down. ConnectionShutdownTotal, + + /// The total number of container events processed. ContainerProcessedEventsTotal, + + /// The total number of times Vector stopped watching for container logs. ContainersUnwatchedTotal, + + /// The total number of times Vector started watching for container logs. ContainersWatchedTotal, + + /// The total number of byte order marks (BOM) removed from incoming data. DecoderBomRemovalsTotal, + + /// The total number of warnings when replacing malformed characters during decoding. DecoderMalformedReplacementWarningsTotal, + + /// The total number of bytes loaded into Doris. DorisBytesLoadedTotal, + + /// The total number of rows filtered by Doris during stream load. DorisRowsFilteredTotal, + + /// The total number of rows successfully loaded into Doris. DorisRowsLoadedTotal, + + /// The total number of warnings when replacing unmappable characters during encoding. EncoderUnmappableReplacementWarningsTotal, + + /// The total number of events discarded by this component. EventsDiscardedTotal, + + /// The total number of files Vector has found to watch. FilesAddedTotal, + + /// The total number of files deleted. FilesDeletedTotal, + + /// The total number of times Vector has resumed watching a file. FilesResumedTotal, + + /// The total number of times Vector has stopped watching a file. FilesUnwatchedTotal, + + /// The total number of gRPC messages received. GrpcServerMessagesReceivedTotal, + + /// The total number of gRPC messages sent. GrpcServerMessagesSentTotal, + + /// The total number of HTTP client errors encountered. HttpClientErrorsTotal, + + /// The total number of sent HTTP requests, tagged with the request method. HttpClientRequestsSentTotal, + + /// The total number of HTTP requests, tagged with the response code. HttpClientResponsesTotal, + + /// The total number of HTTP requests received. HttpServerRequestsReceivedTotal, + + /// The total number of HTTP responses sent. HttpServerResponsesSentTotal, + + /// Total number of message bytes (including framing) received from Kafka brokers. KafkaConsumedMessagesBytesTotal, + + /// Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers. KafkaConsumedMessagesTotal, + + /// Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers. KafkaProducedMessagesBytesTotal, + + /// Total number of messages transmitted (produced) to Kafka brokers. KafkaProducedMessagesTotal, + + /// Total number of bytes transmitted to Kafka brokers. KafkaRequestsBytesTotal, + + /// Total number of requests sent to Kafka brokers. KafkaRequestsTotal, + + /// Total number of bytes received from Kafka brokers. KafkaResponsesBytesTotal, + + /// Total number of responses received from Kafka brokers. KafkaResponsesTotal, + + /// The total number of failed efforts to refresh AWS EC2 metadata. MetadataRefreshFailedTotal, + + /// The total number of AWS EC2 metadata refreshes. MetadataRefreshSuccessfulTotal, + + /// The total number of errors encountered while parsing. ParseErrorsTotal, + + /// The total number of times the Vector instance has quit. QuitTotal, + + /// The total number of times the Vector instance has been reloaded. ReloadedTotal, + + /// The total number of events with rewrapped timestamps. RewrittenTimestampEventsTotal, + + /// The total number of successful deferrals of SQS messages. SqsMessageDeferSucceededTotal, + + /// The total number of successful deletions of SQS messages. SqsMessageDeleteSucceededTotal, + + /// The total number of SQS messages successfully processed. SqsMessageProcessingSucceededTotal, + + /// The total number of times successfully receiving SQS messages. SqsMessageReceiveSucceededTotal, + + /// The total number of received SQS messages. SqsMessageReceivedMessagesTotal, + + /// The number of stale events that Vector has flushed. StaleEventsFlushedTotal, + + /// The total number of times the Vector instance has been started. StartedTotal, + + /// The total number of times the Vector instance has been stopped. StoppedTotal, + + /// The total number of events that contained a tag which exceeded the configured cardinality limit. TagCardinalityUntrackedEventsTotal, + + /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. TagValueLimitExceededTotal, + + /// The total number of times the value limit was reached. ValueLimitReachedTotal, + + /// The total number of bytes sent over WebSocket connections. WebsocketBytesSentTotal, + + /// The total number of messages sent over WebSocket connections. WebsocketMessagesSentTotal, + + /// The total number of times the Windows service has been installed. WindowsServiceInstallTotal, + + /// The total number of times the Windows service has been restarted. WindowsServiceRestartTotal, + + /// The total number of times the Windows service has been started. WindowsServiceStartTotal, + + /// The total number of times the Windows service has been stopped. WindowsServiceStopTotal, + + /// The total number of times the Windows service has been uninstalled. WindowsServiceUninstallTotal, + + /// The total number of failures to annotate Kubernetes events with namespace metadata. K8sEventNamespaceAnnotationFailuresTotal, + + /// The total number of failures to annotate Kubernetes events with node metadata. K8sEventNodeAnnotationFailuresTotal, + + /// The total number of edge cases encountered while picking format of the Kubernetes log message. K8sFormatPickerEdgeCasesTotal, + + /// The total number of failures to parse a message as a JSON object. K8sDockerFormatParseFailuresTotal, + + /// The total number of times an S3 record in an SQS message was ignored. SqsS3EventRecordIgnoredTotal, + + /// The total number of bytes allocated by this component. ComponentAllocatedBytesTotal, + + /// The total number of bytes deallocated by this component. ComponentDeallocatedBytesTotal, + + /// The total number of failed insertions into the in-memory enrichment table. MemoryEnrichmentTableFailedInsertions, + + /// The total number of failed reads from the in-memory enrichment table. MemoryEnrichmentTableFailedReads, + + /// The total number of flushes of the in-memory enrichment table. MemoryEnrichmentTableFlushesTotal, + + /// The total number of successful insertions into the in-memory enrichment table. MemoryEnrichmentTableInsertionsTotal, + + /// The total number of successful reads from the in-memory enrichment table. MemoryEnrichmentTableReadsTotal, + + /// The total number of entries evicted from the in-memory enrichment table due to TTL expiration. MemoryEnrichmentTableTtlExpirations, } +/// Canonical list of all per-component internal histogram metric names emitted by Vector. +#[configurable_component] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] pub enum HistogramName { + /// A histogram of the number of events passed in each internal batch in Vector's internal topology. + /// + /// Note that this is separate than sink-level batching. It is mostly useful for low level + /// debugging performance issues in Vector due to small internal batches. ComponentReceivedEventsCount, + + /// The size in bytes of each event received by the source. ComponentReceivedBytes, + + /// The duration spent sending a payload to this buffer. BufferSendDurationSeconds, + + /// The elapsed time, in fractional seconds, that an event spends in a single transform. + /// + /// This includes both the time spent queued in the transform's input buffer and the time spent + /// executing the transform itself. ComponentLatencySeconds, + + /// The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds. SourceLagTimeSeconds, + + /// The time elapsed blocking on the downstream channel to accept a single chunk from a batch of events received at the source. SourceSendLatencySeconds, + + /// The time elapsed blocking on the downstream channel to accept an entire batch of events received at the source. SourceSendBatchLatencySeconds, + + /// The average round-trip time (RTT) for the current window. AdaptiveConcurrencyAveragedRtt, + + /// The amount of back pressure on the current component. AdaptiveConcurrencyBackPressure, + + /// The number of outbound requests currently awaiting a response. AdaptiveConcurrencyInFlight, + + /// The concurrency limit that the adaptive concurrency feature has decided on for this current window. AdaptiveConcurrencyLimit, + + /// The observed round-trip time (RTT) for requests. AdaptiveConcurrencyObservedRtt, + + /// The mean round-trip time (RTT) for the current window. AdaptiveConcurrencyPastRttMean, + + /// The number of times the concurrency limit was reached. AdaptiveConcurrencyReachedLimit, + + /// The time taken to process an S3 object that succeeded, in seconds. S3ObjectProcessingSucceededDurationSeconds, + + /// The time taken to process an S3 object that failed, in seconds. S3ObjectProcessingFailedDurationSeconds, + + /// The duration spent collecting metrics for this component. CollectDurationSeconds, + + /// The command execution duration in seconds. CommandExecutionDurationSeconds, + + /// The duration spent handling a gRPC request. GrpcServerHandlerDurationSeconds, + + /// The duration spent handling an HTTP request. HttpServerHandlerDurationSeconds, + + /// The round-trip time (RTT) of HTTP requests. HttpClientRttSeconds, + + /// The round-trip time (RTT) of HTTP requests, tagged with the response code. HttpClientResponseRttSeconds, + + /// The round-trip time (RTT) of HTTP requests that resulted in an error. HttpClientErrorRttSeconds, } @@ -164,33 +410,89 @@ impl HistogramName { } } +/// Canonical list of all per-component internal gauge metric names emitted by Vector. +#[configurable_component] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] pub enum GaugeName { + /// The mean elapsed time, in fractional seconds, that an event spends in a single transform. + /// + /// This includes both the time spent queued in the transform's input buffer and the time spent + /// executing the transform itself. This value is smoothed over time using an exponentially + /// weighted moving average (EWMA). ComponentLatencyMeanSeconds, + + /// The maximum number of events in the buffer. BufferMaxSizeEvents, + + /// The maximum size in events that the buffer can store. BufferMaxEventSize, + + /// The maximum number of bytes in the buffer. BufferMaxSizeBytes, + + /// The maximum size in bytes that the buffer can store. BufferMaxByteSize, + + /// The number of events currently in the buffer. BufferEvents, + + /// The number of events currently in the buffer. BufferSizeEvents, + + /// The number of bytes currently in the buffer. BufferSizeBytes, + + /// The number of bytes currently in the buffer. BufferByteSize, + + /// The current utilization of this component, expressed as a value from 0 to 1. Utilization, + + /// The number of bytes currently allocated by this component. ComponentAllocatedBytes, + + /// The total number of open files. OpenFiles, + + /// The number of seconds the Vector instance has been running. UptimeSeconds, + + /// Pseudo-metric that provides build information for the Vector instance. BuildInfo, + + /// Current number of messages in producer queues. KafkaQueueMessages, + + /// Current total size of messages in producer queues. KafkaQueueMessagesBytes, + + /// The Kafka consumer lag. KafkaConsumerLag, + + /// The total memory currently being used by the Lua runtime. LuaMemoryUsedBytes, + + /// The number of current open connections to Vector. OpenConnections, + + /// The number of currently active endpoints. ActiveEndpoints, + + /// The number of outstanding Splunk HEC indexer acknowledgement acks. SplunkPendingAcks, + + /// Number of clients attached to a component. ActiveClients, + + /// The number of objects currently stored in the in-memory enrichment table. MemoryEnrichmentTableObjectsCount, + + /// The total size in bytes of all objects stored in the in-memory enrichment table. MemoryEnrichmentTableByteSize, + + /// The number of tag keys currently being tracked by the tag cardinality limit transform. TagCardinalityTrackedKeys, } From d9e5acb40964d33b3f5c49f2b62f332aa04863ba Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 15 May 2026 16:45:32 -0400 Subject: [PATCH 07/43] chore(metrics): generate metric descriptions from docs.json via jq --- scripts/generate-internal-metric-docs.sh | 30 +++ .../components/sources/internal_metrics.cue | 234 ------------------ .../internal_metric_descriptions.cue | 140 +++++++++++ 3 files changed, 170 insertions(+), 234 deletions(-) create mode 100755 scripts/generate-internal-metric-docs.sh create mode 100644 website/cue/reference/generated/internal_metric_descriptions.cue diff --git a/scripts/generate-internal-metric-docs.sh b/scripts/generate-internal-metric-docs.sh new file mode 100755 index 0000000000000..01da9479792f9 --- /dev/null +++ b/scripts/generate-internal-metric-docs.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Generates website/cue/reference/generated/internal_metric_descriptions.cue +# from website/data/docs.json. +# +# Run this after regenerating docs.json to keep metric descriptions in sync +# with the Rust source (CounterName / HistogramName / GaugeName doc comments). +set -euo pipefail + +DOCS_JSON="${1:-website/data/docs.json}" +OUT="website/cue/reference/generated/internal_metric_descriptions.cue" + +{ + echo 'package metadata' + echo '' + echo '// Auto-generated by scripts/generate-internal-metric-docs.sh' + echo '// Do not edit manually — update the doc comment on the relevant' + echo '// CounterName / HistogramName / GaugeName variant in' + echo '// lib/vector-common/src/internal_event/metric_name.rs instead.' + echo 'components: sources: internal_metrics: output: metrics: {' + jq -r ' + .components.sources.internal_metrics.output.metrics + | to_entries + | sort_by(.key) + | .[] + | "\t\(.key): description: \(.value.description | @json)" + ' "$DOCS_JSON" + echo '}' +} > "$OUT" + +echo "Written: $OUT ($(wc -l < "$OUT") lines)" diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index 1e990965638c6..8aca242493c6b 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -3,10 +3,6 @@ package metadata components: sources: internal_metrics: { title: "Internal Metrics" - description: """ - Exposes Vector's own internal metrics, allowing you to collect, process, - and route Vector's internal metrics just like other metrics. - """ classes: { commonly_used: true @@ -42,12 +38,10 @@ components: sources: internal_metrics: { // Default internal metrics tags _internal_metrics_tags: { pid: { - description: "The process ID of the Vector instance." required: false examples: ["4232"] } host: { - description: "The hostname of the system Vector is running on." required: false examples: [_values.local_host] } @@ -55,85 +49,71 @@ components: sources: internal_metrics: { // Instance-level "process" metrics active_clients: { - description: "Number of clients attached to a component." type: "gauge" default_namespace: "vector" tags: _component_tags } aggregate_events_recorded_total: { - description: "The number of events recorded by the aggregate transform." type: "counter" default_namespace: "vector" tags: _component_tags } aggregate_failed_updates: { - description: "The number of failed metric updates, `incremental` adds, encountered by the aggregate transform." type: "counter" default_namespace: "vector" tags: _component_tags } aggregate_flushes_total: { - description: "The number of flushes done by the aggregate transform." type: "counter" default_namespace: "vector" tags: _component_tags } api_started_total: { - description: "The number of times the Vector API has been started." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } component_timed_out_events_total: { - description: "The total number of events for which this source responded with a timeout error." type: "counter" default_namespace: "vector" tags: _component_tags } component_timed_out_requests_total: { - description: "The total number of requests for which this source responded with a timeout error." type: "counter" default_namespace: "vector" tags: _component_tags } connection_established_total: { - description: "The total number of times a connection has been established." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } connection_send_errors_total: { - description: "The total number of errors sending data via the connection." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } connection_shutdown_total: { - description: "The total number of times the connection has been shut down." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } quit_total: { - description: "The total number of times the Vector instance has quit." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } reloaded_total: { - description: "The total number of times the Vector instance has been reloaded." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } started_total: { - description: "The total number of times the Vector instance has been started." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } stopped_total: { - description: "The total number of times the Vector instance has been stopped." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags @@ -142,37 +122,31 @@ components: sources: internal_metrics: { // Metrics emitted by one or more components // Reusable metric definitions adaptive_concurrency_averaged_rtt: { - description: "The average round-trip time (RTT) for the current window." type: "histogram" default_namespace: "vector" tags: _component_tags } adaptive_concurrency_in_flight: { - description: "The number of outbound requests currently awaiting a response." type: "histogram" default_namespace: "vector" tags: _component_tags } adaptive_concurrency_limit: { - description: "The concurrency limit that the adaptive concurrency feature has decided on for this current window." type: "histogram" default_namespace: "vector" tags: _component_tags } adaptive_concurrency_observed_rtt: { - description: "The observed round-trip time (RTT) for requests." type: "histogram" default_namespace: "vector" tags: _component_tags } checkpoints_total: { - description: "The total number of files checkpointed." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } checksum_errors_total: { - description: "The total number of errors identifying files via checksum." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { @@ -180,31 +154,26 @@ components: sources: internal_metrics: { } } collect_completed_total: { - description: "The total number of metrics collections completed for this component." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } collect_duration_seconds: { - description: "The duration spent collecting of metrics for this component." type: "histogram" default_namespace: "vector" tags: _internal_metrics_tags } command_executed_total: { - description: "The total number of times a command has been executed." type: "counter" default_namespace: "vector" tags: _component_tags } command_execution_duration_seconds: { - description: "The command execution duration in seconds." type: "histogram" default_namespace: "vector" tags: _component_tags } connection_read_errors_total: { - description: "The total number of errors reading datagram." type: "counter" default_namespace: "vector" tags: _component_tags & { @@ -218,55 +187,46 @@ components: sources: internal_metrics: { } } container_processed_events_total: { - description: "The total number of container events processed." type: "counter" default_namespace: "vector" tags: _component_tags } containers_unwatched_total: { - description: "The total number of times Vector stopped watching for container logs." type: "counter" default_namespace: "vector" tags: _component_tags } containers_watched_total: { - description: "The total number of times Vector started watching for container logs." type: "counter" default_namespace: "vector" tags: _component_tags } doris_bytes_loaded_total: { - description: "The total number of bytes loaded into Doris." type: "counter" default_namespace: "vector" tags: _component_tags } doris_rows_filtered_total: { - description: "The total number of rows filtered by Doris during stream load." type: "counter" default_namespace: "vector" tags: _component_tags } doris_rows_loaded_total: { - description: "The total number of rows successfully loaded into Doris." type: "counter" default_namespace: "vector" tags: _component_tags } k8s_format_picker_edge_cases_total: { - description: "The total number of edge cases encountered while picking format of the Kubernetes log message." type: "counter" default_namespace: "vector" tags: _component_tags } k8s_docker_format_parse_failures_total: { - description: "The total number of failures to parse a message as a JSON object." type: "counter" default_namespace: "vector" tags: _component_tags } events_discarded_total: { - description: "The total number of events discarded by this component." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { @@ -274,29 +234,16 @@ components: sources: internal_metrics: { } } component_latency_seconds: { - description: """ - The elapsed time, in fractional seconds, that an event spends in a single transform. - - This includes both the time spent queued in the transform’s input buffer and the time spent executing the transform itself. - """ type: "histogram" default_namespace: "vector" tags: _internal_metrics_tags } component_latency_mean_seconds: { - description: """ - The mean elapsed time, in fractional seconds, that an event spends in a single transform. - - This includes both the time spent queued in the transform’s input buffer and the time spent executing the transform itself. - - This value is smoothed over time using an exponentially weighted moving average (EWMA). - """ type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags } buffer_byte_size: { - description: "The number of bytes currently in the buffer." type: "gauge" default_namespace: "vector" tags: _component_tags @@ -304,7 +251,6 @@ components: sources: internal_metrics: { deprecated_message: "This metric has been deprecated in favor of [`buffer_size_bytes`](#buffer_size_bytes)." } buffer_events: { - description: "The number of events currently in the buffer." type: "gauge" default_namespace: "vector" tags: _component_tags @@ -312,66 +258,55 @@ components: sources: internal_metrics: { deprecated_message: "This metric has been deprecated in favor of [`buffer_size_events`](#buffer_size_events)." } buffer_size_bytes: { - description: "The number of bytes currently in the buffer." type: "gauge" default_namespace: "vector" tags: _component_tags } buffer_size_events: { - description: "The number of events currently in the buffer." type: "gauge" default_namespace: "vector" tags: _component_tags } buffer_discarded_events_total: { - description: "The number of events dropped by this non-blocking buffer." type: "counter" default_namespace: "vector" tags: _component_tags } buffer_received_event_bytes_total: { - description: "The number of bytes received by this buffer." type: "counter" default_namespace: "vector" tags: _component_tags } buffer_received_events_total: { - description: "The number of events received by this buffer." type: "counter" default_namespace: "vector" tags: _component_tags } buffer_send_duration_seconds: { - description: "The duration spent sending a payload to this buffer." type: "histogram" default_namespace: "vector" tags: _component_tags } buffer_sent_event_bytes_total: { - description: "The number of bytes sent by this buffer." type: "counter" default_namespace: "vector" tags: _component_tags } buffer_sent_events_total: { - description: "The number of events sent by this buffer." type: "counter" default_namespace: "vector" tags: _component_tags } component_discarded_events_total: { - description: "The number of events dropped by this component." type: "counter" default_namespace: "vector" tags: _component_tags & { intentional: { - description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error." required: true } } } component_errors_total: { - description: "The total number of errors encountered by this component." type: "counter" default_namespace: "vector" tags: _component_tags & { @@ -380,222 +315,171 @@ components: sources: internal_metrics: { } } component_received_bytes_total: { - description: string | *"The number of raw bytes accepted by this component from source origins." type: "counter" default_namespace: "vector" tags: component_received_events_total.tags } component_received_bytes: { - description: string | *"The size in bytes of each event received by the source." type: "histogram" default_namespace: "vector" tags: component_received_events_total.tags } component_received_events_total: { - description: """ - The number of events accepted by this component either from tagged - origins like file and uri, or cumulatively from other origins. - """ type: "counter" default_namespace: "vector" tags: _component_tags & { file: { - description: "The file from which the data originated." required: false } uri: { - description: "The sanitized URI from which the data originated." required: false } container_name: { - description: "The name of the container from which the data originated." required: false } pod_name: { - description: "The name of the pod from which the data originated." required: false } peer_addr: { - description: "The IP from which the data originated." required: false } peer_path: { - description: "The pathname from which the data originated." required: false } mode: _mode } } component_received_events_count: { - description: """ - A histogram of the number of events passed in each internal batch in Vector's internal topology. - - Note that this is separate than sink-level batching. It is mostly useful for low level debugging - performance issues in Vector due to small internal batches. - """ type: "histogram" default_namespace: "vector" tags: _component_tags & { file: { - description: "The file from which the data originated." required: false } uri: { - description: "The sanitized URI from which the data originated." required: false } container_name: { - description: "The name of the container from which the data originated." required: false } pod_name: { - description: "The name of the pod from which the data originated." required: false } peer_addr: { - description: "The IP from which the data originated." required: false } peer_path: { - description: "The pathname from which the data originated." required: false } mode: _mode } } component_received_event_bytes_total: { - description: """ - The number of event bytes accepted by this component either from - tagged origins like file and uri, or cumulatively from other origins. - """ type: "counter" default_namespace: "vector" tags: component_received_events_total.tags } component_sent_bytes_total: { - description: "The number of raw bytes sent by this component to destination sinks." type: "counter" default_namespace: "vector" tags: _component_tags & { endpoint: { - description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string." required: false } file: { - description: "The absolute path of the destination file." required: false } protocol: { - description: "The protocol used to send the bytes." required: true } region: { - description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname." required: false } } } component_sent_events_total: { - description: "The total number of events emitted by this component." type: "counter" default_namespace: "vector" tags: _component_tags & {output: _output} } component_sent_event_bytes_total: { - description: "The total number of event bytes emitted by this component." type: "counter" default_namespace: "vector" tags: _component_tags & {output: _output} } internal_metrics_cardinality: { - description: "The total number of metrics emitted from the internal metrics registry." type: "gauge" default_namespace: "vector" tags: {} } internal_metrics_cardinality_total: { - description: "The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`." type: "counter" default_namespace: "vector" tags: internal_metrics_cardinality.tags } kafka_queue_messages: { - description: "Current number of messages in producer queues." type: "gauge" default_namespace: "vector" tags: _component_tags } kafka_queue_messages_bytes: { - description: "Current total size of messages in producer queues." type: "gauge" default_namespace: "vector" tags: _component_tags } kafka_requests_total: { - description: "Total number of requests sent to Kafka brokers." type: "counter" default_namespace: "vector" tags: _component_tags } kafka_requests_bytes_total: { - description: "Total number of bytes transmitted to Kafka brokers." type: "counter" default_namespace: "vector" tags: _component_tags } kafka_responses_total: { - description: "Total number of responses received from Kafka brokers." type: "counter" default_namespace: "vector" tags: _component_tags } kafka_responses_bytes_total: { - description: "Total number of bytes received from Kafka brokers." type: "counter" default_namespace: "vector" tags: _component_tags } kafka_produced_messages_total: { - description: "Total number of messages transmitted (produced) to Kafka brokers." type: "counter" default_namespace: "vector" tags: _component_tags } kafka_produced_messages_bytes_total: { - description: "Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers." type: "counter" default_namespace: "vector" tags: _component_tags } kafka_consumed_messages_total: { - description: "Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers." type: "counter" default_namespace: "vector" tags: _component_tags } kafka_consumed_messages_bytes_total: { - description: "Total number of message bytes (including framing) received from Kafka brokers." type: "counter" default_namespace: "vector" tags: _component_tags } kafka_consumer_lag: { - description: "The Kafka consumer lag." type: "gauge" default_namespace: "vector" tags: _component_tags & { topic_id: { - description: "The Kafka topic id." required: true } partition_id: { - description: "The Kafka partition id." required: true } } } files_added_total: { - description: "The total number of files Vector has found to watch." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { @@ -603,7 +487,6 @@ components: sources: internal_metrics: { } } files_deleted_total: { - description: "The total number of files deleted." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { @@ -611,7 +494,6 @@ components: sources: internal_metrics: { } } files_resumed_total: { - description: "The total number of times Vector has resumed watching a file." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { @@ -619,7 +501,6 @@ components: sources: internal_metrics: { } } files_unwatched_total: { - description: "The total number of times Vector has stopped watching a file." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { @@ -627,13 +508,11 @@ components: sources: internal_metrics: { } } open_files: { - description: "The total number of open files." type: "counter" default_namespace: "vector" tags: _component_tags } grpc_server_messages_received_total: { - description: "The total number of gRPC messages received." type: "counter" default_namespace: "vector" tags: _component_tags & { @@ -642,7 +521,6 @@ components: sources: internal_metrics: { } } grpc_server_messages_sent_total: { - description: "The total number of gRPC messages sent." type: "counter" default_namespace: "vector" tags: _component_tags & { @@ -652,7 +530,6 @@ components: sources: internal_metrics: { } } grpc_server_handler_duration_seconds: { - description: "The duration spent handling a gRPC request." type: "histogram" default_namespace: "vector" tags: _component_tags & { @@ -662,7 +539,6 @@ components: sources: internal_metrics: { } } http_client_response_rtt_seconds: { - description: "The round-trip time (RTT) of HTTP requests, tagged with the response code." type: "histogram" default_namespace: "vector" tags: _component_tags & { @@ -670,7 +546,6 @@ components: sources: internal_metrics: { } } http_client_requests_sent_total: { - description: "The total number of sent HTTP requests, tagged with the request method." type: "counter" default_namespace: "vector" tags: _component_tags & { @@ -678,7 +553,6 @@ components: sources: internal_metrics: { } } http_client_responses_total: { - description: "The total number of HTTP requests, tagged with the response code." type: "counter" default_namespace: "vector" tags: _component_tags & { @@ -686,19 +560,16 @@ components: sources: internal_metrics: { } } http_client_rtt_seconds: { - description: "The round-trip time (RTT) of HTTP requests." type: "histogram" default_namespace: "vector" tags: _component_tags } http_requests_total: { - description: "The total number of HTTP requests issued by this component." type: "counter" default_namespace: "vector" tags: _component_tags } http_server_requests_received_total: { - description: "The total number of HTTP requests received." type: "counter" default_namespace: "vector" tags: _component_tags & { @@ -707,7 +578,6 @@ components: sources: internal_metrics: { } } http_server_responses_sent_total: { - description: "The total number of HTTP responses sent." type: "counter" default_namespace: "vector" tags: _component_tags & { @@ -717,7 +587,6 @@ components: sources: internal_metrics: { } } http_server_handler_duration_seconds: { - description: "The duration spent handling a HTTP request." type: "histogram" default_namespace: "vector" tags: _component_tags & { @@ -727,67 +596,56 @@ components: sources: internal_metrics: { } } invalid_record_total: { - description: "The total number of invalid records that have been discarded." type: "counter" default_namespace: "vector" tags: _component_tags } lua_memory_used_bytes: { - description: "The total memory currently being used by the Lua runtime." type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags } metadata_refresh_failed_total: { - description: "The total number of failed efforts to refresh AWS EC2 metadata." type: "counter" default_namespace: "vector" tags: _component_tags } metadata_refresh_successful_total: { - description: "The total number of AWS EC2 metadata refreshes." type: "counter" default_namespace: "vector" tags: _component_tags } open_connections: { - description: "The number of current open connections to Vector." type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags } protobuf_decode_errors_total: { - description: "The total number of [Protocol Buffers](\(urls.protobuf)) errors thrown during communication between Vector instances." type: "counter" default_namespace: "vector" tags: _component_tags } send_errors_total: { - description: "The total number of errors sending messages." type: "counter" default_namespace: "vector" tags: _component_tags } source_lag_time_seconds: { - description: "The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds." type: "histogram" default_namespace: "vector" tags: _component_tags } source_send_batch_latency_seconds: { - description: "The time elapsed blocking on the downstream channel to accept an entire batch of events received at the source" type: "histogram" default_namespace: "vector" tags: _component_tags } source_send_latency_seconds: { - description: "The time elapsed blocking on the downstream channel to accept a single chunk from a batch of events received at the source" type: "histogram" default_namespace: "vector" tags: _component_tags } source_buffer_max_byte_size: { - description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -797,7 +655,6 @@ components: sources: internal_metrics: { deprecated_message: "This metric has been deprecated in favor of [`source_buffer_max_size_bytes`](#source_buffer_max_size_bytes)." } source_buffer_max_event_size: { - description: "The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -807,7 +664,6 @@ components: sources: internal_metrics: { deprecated_message: "This metric has been deprecated in favor of [`source_buffer_max_size_events`](#source_buffer_max_size_events)." } source_buffer_max_size_bytes: { - description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -815,7 +671,6 @@ components: sources: internal_metrics: { } } source_buffer_max_size_events: { - description: "The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -823,7 +678,6 @@ components: sources: internal_metrics: { } } source_buffer_utilization: { - description: "The utilization level of the source buffer. The outputs of the source send data to this buffer." type: "histogram" default_namespace: "vector" tags: _component_tags & { @@ -831,7 +685,6 @@ components: sources: internal_metrics: { } } source_buffer_utilization_level: { - description: "The current utilization level of the source buffer. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -839,7 +692,6 @@ components: sources: internal_metrics: { } } source_buffer_utilization_mean: { - description: "The mean utilization level of the source buffer. The outputs of the source send data to this buffer. The mean utilization is smoothed over time using an exponentially weighted moving average (EWMA)." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -847,71 +699,59 @@ components: sources: internal_metrics: { } } splunk_pending_acks: { - description: "The number of outstanding Splunk HEC indexer acknowledgement acks." type: "gauge" default_namespace: "vector" tags: _component_tags } streams_total: { - description: "The total number of streams." type: "counter" default_namespace: "vector" tags: _component_tags } s3_object_processing_failed_duration_seconds: { - description: "The time taken to process an S3 object that failed, in seconds." type: "histogram" default_namespace: "vector" tags: _component_tags & { bucket: { - description: "The name of the S3 bucket." required: true } } } s3_object_processing_succeeded_duration_seconds: { - description: "The time taken to process an S3 object that succeeded, in seconds." type: "histogram" default_namespace: "vector" tags: _component_tags & { bucket: { - description: "The name of the S3 bucket." required: true } } } sqs_message_delete_succeeded_total: { - description: "The total number of successful deletions of SQS messages." type: "counter" default_namespace: "vector" tags: _component_tags } sqs_message_processing_succeeded_total: { - description: "The total number of SQS messages successfully processed." type: "counter" default_namespace: "vector" tags: _component_tags } sqs_message_receive_succeeded_total: { - description: "The total number of times successfully receiving SQS messages." type: "counter" default_namespace: "vector" tags: _component_tags } sqs_message_received_messages_total: { - description: "The total number of received SQS messages." type: "counter" default_namespace: "vector" tags: _component_tags } sqs_s3_event_record_ignored_total: { - description: "The total number of times an S3 record in an SQS message was ignored (for an event that was not `ObjectCreated`)." type: "counter" default_namespace: "vector" tags: _component_tags & { ignore_type: { - description: "The reason for ignoring the S3 record" required: true enum: { "invalid_event_kind": "The kind of invalid event." @@ -920,52 +760,33 @@ components: sources: internal_metrics: { } } stale_events_flushed_total: { - description: "The number of stale events that Vector has flushed." type: "counter" default_namespace: "vector" tags: _component_tags } stdin_reads_failed_total: { - description: "The total number of errors reading from stdin." type: "counter" default_namespace: "vector" tags: _component_tags } tag_value_limit_exceeded_total: { - description: """ - The total number of events discarded because the tag has been rejected after - hitting the configured `value_limit`. When `internal_metrics.include_extended_tags` - is enabled in the `tag_cardinality_limit` transform, this metric includes - `metric_name` and `tag_key` labels. By default, this metric has no labels to - keep cardinality low. - """ type: "counter" default_namespace: "vector" tags: _component_tags & { metric_name: { - description: """ - The name of the metric whose tag value limit was exceeded. - Only present when `internal_metrics.include_extended_tags` is enabled. - """ required: false } tag_key: { - description: """ - The key of the tag whose value limit was exceeded. - Only present when `internal_metrics.include_extended_tags` is enabled. - """ required: false } } } timestamp_parse_errors_total: { - description: "The total number of errors encountered parsing [RFC 3339](\(urls.rfc_3339)) timestamps." type: "counter" default_namespace: "vector" tags: _component_tags } transform_buffer_max_byte_size: { - description: "The maximum number of bytes the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -975,7 +796,6 @@ components: sources: internal_metrics: { deprecated_message: "This metric has been deprecated in favor of [`transform_buffer_max_size_bytes`](#transform_buffer_max_size_bytes)." } transform_buffer_max_event_size: { - description: "The maximum number of events the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -985,7 +805,6 @@ components: sources: internal_metrics: { deprecated_message: "This metric has been deprecated in favor of [`transform_buffer_max_size_events`](#transform_buffer_max_size_events)." } transform_buffer_max_size_bytes: { - description: "The maximum number of bytes the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -993,7 +812,6 @@ components: sources: internal_metrics: { } } transform_buffer_max_size_events: { - description: "The maximum number of events the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -1001,7 +819,6 @@ components: sources: internal_metrics: { } } transform_buffer_utilization: { - description: "The utilization level of the buffer that feeds into a transform." type: "histogram" default_namespace: "vector" tags: _component_tags & { @@ -1009,7 +826,6 @@ components: sources: internal_metrics: { } } transform_buffer_utilization_level: { - description: "The current utilization level of the buffer that feeds into a transform." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -1017,7 +833,6 @@ components: sources: internal_metrics: { } } transform_buffer_utilization_mean: { - description: "The mean utilization level of the buffer that feeds into a transform. This value is smoothed over time using an exponentially weighted moving average (EWMA)." type: "gauge" default_namespace: "vector" tags: _component_tags & { @@ -1025,18 +840,15 @@ components: sources: internal_metrics: { } } uptime_seconds: { - description: "The total number of seconds the Vector instance has been up." type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags } utf8_convert_errors_total: { - description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." type: "counter" default_namespace: "vector" tags: _component_tags & { mode: { - description: "The connection mode used by the component." required: true enum: { udp: "User Datagram Protocol" @@ -1045,43 +857,32 @@ components: sources: internal_metrics: { } } utilization: { - description: "A ratio from 0 to 1 of the load on a component. A value of 0 would indicate a completely idle component that is simply waiting for input. A value of 1 would indicate a that is never idle. This value is updated every 5 seconds." type: "gauge" default_namespace: "vector" tags: _component_tags } build_info: { - description: "Has a fixed value of 1.0. Contains build information such as Rust and Vector versions." type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags & { debug: { - description: "Whether this is a debug build of Vector" required: true } version: { - description: "Vector version." required: true } rust_version: { - description: "The Rust version from the package manifest." required: true } arch: { - description: "The target architecture being compiled for. (e.g. x86_64)" required: true } revision: { - description: "Revision identifer, related to versioned releases." required: true } } } value_limit_reached_total: { - description: """ - The total number of times new values for a key have been rejected because the - value limit has been reached. - """ type: "counter" default_namespace: "vector" tags: _component_tags @@ -1089,41 +890,26 @@ components: sources: internal_metrics: { // Windows metrics windows_service_install_total: { - description: """ - The total number of times the Windows service has been installed. - """ type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } windows_service_restart_total: { - description: """ - The total number of times the Windows service has been restarted. - """ type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } windows_service_start_total: { - description: """ - The total number of times the Windows service has been started. - """ type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } windows_service_stop_total: { - description: """ - The total number of times the Windows service has been stopped. - """ type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } windows_service_uninstall_total: { - description: """ - The total number of times the Windows service has been uninstalled. - """ type: "counter" default_namespace: "vector" tags: _internal_metrics_tags @@ -1131,7 +917,6 @@ components: sources: internal_metrics: { // config metrics config_reload_rejected: { - description: "Number of configuration reload attempts that were rejected." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { @@ -1139,7 +924,6 @@ components: sources: internal_metrics: { } } config_reloaded: { - description: "Number of times a new configuration was loaded successfully." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags @@ -1154,11 +938,9 @@ components: sources: internal_metrics: { // All available tags _collector: { - description: "Which collector this metric comes from." required: true } _component_kind: { - description: "The Vector component kind." required: true enum: { "sink": "Vector sink components" @@ -1167,22 +949,18 @@ components: sources: internal_metrics: { } } _component_id: { - description: "The Vector component ID." required: true examples: ["my_source", "my_sink"] } _component_type: { - description: "The Vector component type." required: true examples: ["file", "http", "honeycomb", "splunk_hec"] } _endpoint: { - description: "The absolute path of originating file." required: true examples: ["http://localhost:8080/server-status?auto"] } _error_type: { - description: "The type of the error" required: true enum: { "acknowledgements_failed": "The acknowledgement operation failed." @@ -1211,28 +989,22 @@ components: sources: internal_metrics: { } } _file: { - description: "The file that produced the error" required: false } _grpc_method: { - description: "The name of the method called on the gRPC service." required: true } _grpc_service: { - description: "The gRPC service name." required: true } _grpc_status: { - description: "The human-readable [gRPC status code](\(urls.grpc_status_code))." required: true } _host: { - description: "The hostname of the originating system." required: true examples: [_values.local_host] } _mode: { - description: "The connection mode used by the component." required: false enum: { udp: "User Datagram Protocol" @@ -1241,11 +1013,9 @@ components: sources: internal_metrics: { } } _output: { - description: "The specific output of the component." required: false } _stage: { - description: "The stage within the component at which the error occurred." required: true enum: { receiving: "While receiving data." @@ -1254,19 +1024,15 @@ components: sources: internal_metrics: { } } _status: { - description: "The HTTP status code of the request." required: false } _method: { - description: "The HTTP method of the request." required: false } _path: { - description: "The path that produced the error." required: true } _reason: { - description: "The type of the error" required: true enum: { "out_of_order": "The event was out of order." diff --git a/website/cue/reference/generated/internal_metric_descriptions.cue b/website/cue/reference/generated/internal_metric_descriptions.cue new file mode 100644 index 0000000000000..5b4b4102bd720 --- /dev/null +++ b/website/cue/reference/generated/internal_metric_descriptions.cue @@ -0,0 +1,140 @@ +package metadata + +// Auto-generated by scripts/generate-internal-metric-docs.sh +// Do not edit manually — update the doc comment on the relevant +// CounterName / HistogramName / GaugeName variant in +// lib/vector-common/src/internal_event/metric_name.rs instead. +components: sources: internal_metrics: output: metrics: { + active_clients: description: "Number of clients attached to a component." + adaptive_concurrency_averaged_rtt: description: "The average round-trip time (RTT) for the current window." + adaptive_concurrency_in_flight: description: "The number of outbound requests currently awaiting a response." + adaptive_concurrency_limit: description: "The concurrency limit that the adaptive concurrency feature has decided on for this current window." + adaptive_concurrency_observed_rtt: description: "The observed round-trip time (RTT) for requests." + aggregate_events_recorded_total: description: "The number of events recorded by the aggregate transform." + aggregate_failed_updates: description: "The number of failed metric updates, `incremental` adds, encountered by the aggregate transform." + aggregate_flushes_total: description: "The number of flushes done by the aggregate transform." + api_started_total: description: "The number of times the Vector API has been started." + buffer_byte_size: description: "The number of bytes currently in the buffer." + buffer_discarded_events_total: description: "The number of events dropped by this non-blocking buffer." + buffer_events: description: "The number of events currently in the buffer." + buffer_received_event_bytes_total: description: "The number of bytes received by this buffer." + buffer_received_events_total: description: "The number of events received by this buffer." + buffer_send_duration_seconds: description: "The duration spent sending a payload to this buffer." + buffer_sent_event_bytes_total: description: "The number of bytes sent by this buffer." + buffer_sent_events_total: description: "The number of events sent by this buffer." + buffer_size_bytes: description: "The number of bytes currently in the buffer." + buffer_size_events: description: "The number of events currently in the buffer." + build_info: description: "Has a fixed value of 1.0. Contains build information such as Rust and Vector versions." + checkpoints_total: description: "The total number of files checkpointed." + checksum_errors_total: description: "The total number of errors identifying files via checksum." + collect_completed_total: description: "The total number of metrics collections completed for this component." + collect_duration_seconds: description: "The duration spent collecting of metrics for this component." + command_executed_total: description: "The total number of times a command has been executed." + command_execution_duration_seconds: description: "The command execution duration in seconds." + component_discarded_events_total: description: "The number of events dropped by this component." + component_errors_total: description: "The total number of errors encountered by this component." + component_latency_mean_seconds: description: "The mean elapsed time, in fractional seconds, that an event spends in a single transform.\n\nThis includes both the time spent queued in the transform’s input buffer and the time spent executing the transform itself.\n\nThis value is smoothed over time using an exponentially weighted moving average (EWMA)." + component_latency_seconds: description: "The elapsed time, in fractional seconds, that an event spends in a single transform.\n\nThis includes both the time spent queued in the transform’s input buffer and the time spent executing the transform itself." + component_received_bytes: description: "The size in bytes of each event received by the source." + component_received_bytes_total: description: "The number of raw bytes accepted by this component from source origins." + component_received_event_bytes_total: description: "The number of event bytes accepted by this component either from\ntagged origins like file and uri, or cumulatively from other origins." + component_received_events_count: description: "A histogram of the number of events passed in each internal batch in Vector's internal topology.\n\nNote that this is separate than sink-level batching. It is mostly useful for low level debugging\nperformance issues in Vector due to small internal batches." + component_received_events_total: description: "The number of events accepted by this component either from tagged\norigins like file and uri, or cumulatively from other origins." + component_sent_bytes_total: description: "The number of raw bytes sent by this component to destination sinks." + component_sent_event_bytes_total: description: "The total number of event bytes emitted by this component." + component_sent_events_total: description: "The total number of events emitted by this component." + component_timed_out_events_total: description: "The total number of events for which this source responded with a timeout error." + component_timed_out_requests_total: description: "The total number of requests for which this source responded with a timeout error." + config_reload_rejected: description: "Number of configuration reload attempts that were rejected." + config_reloaded: description: "Number of times a new configuration was loaded successfully." + connection_established_total: description: "The total number of times a connection has been established." + connection_read_errors_total: description: "The total number of errors reading datagram." + connection_send_errors_total: description: "The total number of errors sending data via the connection." + connection_shutdown_total: description: "The total number of times the connection has been shut down." + container_processed_events_total: description: "The total number of container events processed." + containers_unwatched_total: description: "The total number of times Vector stopped watching for container logs." + containers_watched_total: description: "The total number of times Vector started watching for container logs." + doris_bytes_loaded_total: description: "The total number of bytes loaded into Doris." + doris_rows_filtered_total: description: "The total number of rows filtered by Doris during stream load." + doris_rows_loaded_total: description: "The total number of rows successfully loaded into Doris." + events_discarded_total: description: "The total number of events discarded by this component." + files_added_total: description: "The total number of files Vector has found to watch." + files_deleted_total: description: "The total number of files deleted." + files_resumed_total: description: "The total number of times Vector has resumed watching a file." + files_unwatched_total: description: "The total number of times Vector has stopped watching a file." + grpc_server_handler_duration_seconds: description: "The duration spent handling a gRPC request." + grpc_server_messages_received_total: description: "The total number of gRPC messages received." + grpc_server_messages_sent_total: description: "The total number of gRPC messages sent." + http_client_requests_sent_total: description: "The total number of sent HTTP requests, tagged with the request method." + http_client_response_rtt_seconds: description: "The round-trip time (RTT) of HTTP requests, tagged with the response code." + http_client_responses_total: description: "The total number of HTTP requests, tagged with the response code." + http_client_rtt_seconds: description: "The round-trip time (RTT) of HTTP requests." + http_requests_total: description: "The total number of HTTP requests issued by this component." + http_server_handler_duration_seconds: description: "The duration spent handling a HTTP request." + http_server_requests_received_total: description: "The total number of HTTP requests received." + http_server_responses_sent_total: description: "The total number of HTTP responses sent." + internal_metrics_cardinality: description: "The total number of metrics emitted from the internal metrics registry." + internal_metrics_cardinality_total: description: "The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`." + invalid_record_total: description: "The total number of invalid records that have been discarded." + k8s_docker_format_parse_failures_total: description: "The total number of failures to parse a message as a JSON object." + k8s_format_picker_edge_cases_total: description: "The total number of edge cases encountered while picking format of the Kubernetes log message." + kafka_consumed_messages_bytes_total: description: "Total number of message bytes (including framing) received from Kafka brokers." + kafka_consumed_messages_total: description: "Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers." + kafka_consumer_lag: description: "The Kafka consumer lag." + kafka_produced_messages_bytes_total: description: "Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers." + kafka_produced_messages_total: description: "Total number of messages transmitted (produced) to Kafka brokers." + kafka_queue_messages: description: "Current number of messages in producer queues." + kafka_queue_messages_bytes: description: "Current total size of messages in producer queues." + kafka_requests_bytes_total: description: "Total number of bytes transmitted to Kafka brokers." + kafka_requests_total: description: "Total number of requests sent to Kafka brokers." + kafka_responses_bytes_total: description: "Total number of bytes received from Kafka brokers." + kafka_responses_total: description: "Total number of responses received from Kafka brokers." + lua_memory_used_bytes: description: "The total memory currently being used by the Lua runtime." + metadata_refresh_failed_total: description: "The total number of failed efforts to refresh AWS EC2 metadata." + metadata_refresh_successful_total: description: "The total number of AWS EC2 metadata refreshes." + open_connections: description: "The number of current open connections to Vector." + open_files: description: "The total number of open files." + protobuf_decode_errors_total: description: "The total number of [Protocol Buffers](https://developers.google.com/protocol-buffers) errors thrown during communication between Vector instances." + quit_total: description: "The total number of times the Vector instance has quit." + reloaded_total: description: "The total number of times the Vector instance has been reloaded." + s3_object_processing_failed_duration_seconds: description: "The time taken to process an S3 object that failed, in seconds." + s3_object_processing_succeeded_duration_seconds: description: "The time taken to process an S3 object that succeeded, in seconds." + send_errors_total: description: "The total number of errors sending messages." + source_buffer_max_byte_size: description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." + source_buffer_max_event_size: description: "The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer." + source_buffer_max_size_bytes: description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." + source_buffer_max_size_events: description: "The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer." + source_buffer_utilization: description: "The utilization level of the source buffer. The outputs of the source send data to this buffer." + source_buffer_utilization_level: description: "The current utilization level of the source buffer. The outputs of the source send data to this buffer." + source_buffer_utilization_mean: description: "The mean utilization level of the source buffer. The outputs of the source send data to this buffer. The mean utilization is smoothed over time using an exponentially weighted moving average (EWMA)." + source_lag_time_seconds: description: "The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds." + splunk_pending_acks: description: "The number of outstanding Splunk HEC indexer acknowledgement acks." + sqs_message_delete_succeeded_total: description: "The total number of successful deletions of SQS messages." + sqs_message_processing_succeeded_total: description: "The total number of SQS messages successfully processed." + sqs_message_receive_succeeded_total: description: "The total number of times successfully receiving SQS messages." + sqs_message_received_messages_total: description: "The total number of received SQS messages." + sqs_s3_event_record_ignored_total: description: "The total number of times an S3 record in an SQS message was ignored (for an event that was not `ObjectCreated`)." + stale_events_flushed_total: description: "The number of stale events that Vector has flushed." + started_total: description: "The total number of times the Vector instance has been started." + stdin_reads_failed_total: description: "The total number of errors reading from stdin." + stopped_total: description: "The total number of times the Vector instance has been stopped." + streams_total: description: "The total number of streams." + tag_value_limit_exceeded_total: description: "The total number of events discarded because the tag has been rejected after\nhitting the configured `value_limit`. When `internal_metrics.include_extended_tags`\nis enabled in the `tag_cardinality_limit` transform, this metric includes\n`metric_name` and `tag_key` labels. By default, this metric has no labels to\nkeep cardinality low." + timestamp_parse_errors_total: description: "The total number of errors encountered parsing [RFC 3339](https://tools.ietf.org/html/rfc3339) timestamps." + transform_buffer_max_byte_size: description: "The maximum number of bytes the buffer that feeds into a transform can hold." + transform_buffer_max_event_size: description: "The maximum number of events the buffer that feeds into a transform can hold." + transform_buffer_max_size_bytes: description: "The maximum number of bytes the buffer that feeds into a transform can hold." + transform_buffer_max_size_events: description: "The maximum number of events the buffer that feeds into a transform can hold." + transform_buffer_utilization: description: "The utilization level of the buffer that feeds into a transform." + transform_buffer_utilization_level: description: "The current utilization level of the buffer that feeds into a transform." + transform_buffer_utilization_mean: description: "The mean utilization level of the buffer that feeds into a transform. This value is smoothed over time using an exponentially weighted moving average (EWMA)." + uptime_seconds: description: "The total number of seconds the Vector instance has been up." + utf8_convert_errors_total: description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." + utilization: description: "A ratio from 0 to 1 of the load on a component. A value of 0 would indicate a completely idle component that is simply waiting for input. A value of 1 would indicate a that is never idle. This value is updated every 5 seconds." + value_limit_reached_total: description: "The total number of times new values for a key have been rejected because the\nvalue limit has been reached." + windows_service_install_total: description: "The total number of times the Windows service has been installed." + windows_service_restart_total: description: "The total number of times the Windows service has been restarted." + windows_service_start_total: description: "The total number of times the Windows service has been started." + windows_service_stop_total: description: "The total number of times the Windows service has been stopped." + windows_service_uninstall_total: description: "The total number of times the Windows service has been uninstalled." +} From dc445a582c6baeb29494b58fed6b2fe2cdd2f7ac Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 15 May 2026 16:55:56 -0400 Subject: [PATCH 08/43] chore(metrics): generate internal metric descriptions via vdev build component-docs --- scripts/generate-internal-metric-docs.sh | 30 -------- src/generate_schema.rs | 22 +++++- .../commands/build/component_docs/runner.rs | 72 +++++++++++++++++++ 3 files changed, 93 insertions(+), 31 deletions(-) delete mode 100755 scripts/generate-internal-metric-docs.sh diff --git a/scripts/generate-internal-metric-docs.sh b/scripts/generate-internal-metric-docs.sh deleted file mode 100755 index 01da9479792f9..0000000000000 --- a/scripts/generate-internal-metric-docs.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Generates website/cue/reference/generated/internal_metric_descriptions.cue -# from website/data/docs.json. -# -# Run this after regenerating docs.json to keep metric descriptions in sync -# with the Rust source (CounterName / HistogramName / GaugeName doc comments). -set -euo pipefail - -DOCS_JSON="${1:-website/data/docs.json}" -OUT="website/cue/reference/generated/internal_metric_descriptions.cue" - -{ - echo 'package metadata' - echo '' - echo '// Auto-generated by scripts/generate-internal-metric-docs.sh' - echo '// Do not edit manually — update the doc comment on the relevant' - echo '// CounterName / HistogramName / GaugeName variant in' - echo '// lib/vector-common/src/internal_event/metric_name.rs instead.' - echo 'components: sources: internal_metrics: output: metrics: {' - jq -r ' - .components.sources.internal_metrics.output.metrics - | to_entries - | sort_by(.key) - | .[] - | "\t\(.key): description: \(.value.description | @json)" - ' "$DOCS_JSON" - echo '}' -} > "$OUT" - -echo "Written: $OUT ($(wc -l < "$OUT") lines)" diff --git a/src/generate_schema.rs b/src/generate_schema.rs index 72204e3945d91..623a6f187948e 100644 --- a/src/generate_schema.rs +++ b/src/generate_schema.rs @@ -3,6 +3,8 @@ use std::{fs, path::PathBuf}; use clap::Parser; +use serde_json::{Value, json}; +use vector_common::internal_event::{CounterName, GaugeName, HistogramName}; use vector_lib::configurable::schema::generate_root_schema; use crate::config::ConfigBuilder; @@ -16,11 +18,29 @@ pub struct Opts { pub(crate) output_path: Option, } +fn metric_enum_schema() -> Value { + generate_root_schema::() + .map(|s| serde_json::to_value(s).unwrap_or(Value::Null)) + .unwrap_or(Value::Null) +} + /// Execute the `generate-schema` command. #[allow(clippy::print_stdout, clippy::print_stderr)] pub fn cmd(opts: &Opts) -> exitcode::ExitCode { match generate_root_schema::() { - Ok(schema) => { + Ok(config_schema) => { + // Convert to Value so we can inject the metric enum schemas. + let mut schema = serde_json::to_value(config_schema) + .expect("rendering root schema to JSON should not fail"); + + // Inject metric name enum schemas so vdev can generate + // internal_metrics.cue output descriptions from them. + schema["_metric_schemas"] = json!({ + "counters": metric_enum_schema::(), + "histograms": metric_enum_schema::(), + "gauges": metric_enum_schema::(), + }); + let json = serde_json::to_string_pretty(&schema) .expect("rendering root schema to JSON should not fail"); diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index 7b9eb3dbbfbac..aa688ca6b459a 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -71,6 +71,12 @@ pub fn run(schema_path: &Path) -> Result<()> { // is now rendered as a top-level field with `group: "api"`. render_and_import_generated_top_level_config_schema(&mut context, &root_schema)?; + // 4. Generate internal_metric_descriptions.cue from the metric name enums + // injected into the schema by `vector generate-schema`. + if let Some(metric_schemas) = root_schema.get("_metric_schemas") { + generate_internal_metric_descriptions(metric_schemas)?; + } + Ok(()) } @@ -354,3 +360,69 @@ fn render_and_import_generated_top_level_config_schema( ); Ok(()) } + +/// Extracts variant descriptions from a metric enum JSON schema (oneOf / enum array) +/// and returns an iterator of `(metric_name, description)` pairs. +fn extract_metric_variants(schema: &Value) -> impl Iterator { + schema + .get("oneOf") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|variant| { + let name = variant.get("const").and_then(Value::as_str)?; + let desc = variant.get("description").and_then(Value::as_str)?; + Some((name, desc)) + }) +} + +fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { + let out_path = PathBuf::from( + "website/cue/reference/generated/internal_metric_descriptions.cue", + ); + + if let Some(parent) = out_path.parent() { + fs::create_dir_all(parent)?; + } + + let mut lines: Vec<(String, String)> = Vec::new(); + + for (metric_type, schema) in [ + ("counter", metric_schemas.get("counters")), + ("histogram", metric_schemas.get("histograms")), + ("gauge", metric_schemas.get("gauges")), + ] { + if let Some(schema) = schema { + for (name, desc) in extract_metric_variants(schema) { + // Escape backslashes and double-quotes for CUE string literals. + let escaped = desc.replace('\\', "\\\\").replace('"', "\\\""); + lines.push((name.to_string(), escaped)); + let _ = metric_type; // type is encoded in which enum the variant came from + } + } + } + + lines.sort_by(|a, b| a.0.cmp(&b.0)); + + let mut cue = String::from( + "package metadata\n\ + \n\ + // Auto-generated by `vdev build component-docs`.\n\ + // Do not edit manually — update the doc comment on the relevant\n\ + // CounterName / HistogramName / GaugeName variant in\n\ + // lib/vector-common/src/internal_event/metric_name.rs instead,\n\ + // then re-run `make generate-component-docs`.\n\ + components: sources: internal_metrics: output: metrics: {\n", + ); + for (name, desc) in &lines { + cue.push_str(&format!("\t{name}: description: \"{desc}\"\n")); + } + cue.push_str("}\n"); + + fs::write(&out_path, &cue)?; + info!( + "[✓] Wrote internal metric descriptions to '{}'.", + out_path.display() + ); + Ok(()) +} From 1f26375ae35f10c3fa0808a3f9e2d0428051edca Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 10:49:10 -0400 Subject: [PATCH 09/43] fix(vdev): use write! instead of push_str(&format!()) to satisfy clippy --- vdev/src/commands/build/component_docs/runner.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index f83fcca98e2b0..6635dfce56bd0 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -2,6 +2,7 @@ use super::schema::SchemaContext; use anyhow::{Context, Result, bail}; use indexmap::IndexMap; use serde_json::{Value, json}; +use std::fmt::Write as _; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -433,19 +434,16 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { for e in &entries { if e.deprecated { - cue.push_str(&format!("\t{}: {{\n", e.name)); - cue.push_str(&format!("\t\tdescription: \"{}\"\n", e.description)); + writeln!(cue, "\t{}: {{", e.name).unwrap(); + writeln!(cue, "\t\tdescription: \"{}\"", e.description).unwrap(); cue.push_str("\t\tdeprecated: true\n"); if let Some(msg) = &e.deprecated_message { let escaped = msg.replace('\\', "\\\\").replace('"', "\\\""); - cue.push_str(&format!("\t\tdeprecated_message: \"{escaped}\"\n")); + writeln!(cue, "\t\tdeprecated_message: \"{escaped}\"").unwrap(); } cue.push_str("\t}\n"); } else { - cue.push_str(&format!( - "\t{}: description: \"{}\"\n", - e.name, e.description - )); + writeln!(cue, "\t{}: description: \"{}\"", e.name, e.description).unwrap(); } } From b99838ff262e1e7134c38c2684d05ed1cce79f31 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 10:54:08 -0400 Subject: [PATCH 10/43] fix(docs): restore internal_metrics source description removed by mistake --- website/cue/reference/components/sources/internal_metrics.cue | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index 35c7ca9a22cd9..bb4cdfcce20d3 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -3,6 +3,10 @@ package metadata components: sources: internal_metrics: { title: "Internal Metrics" + description: """ + Exposes Vector's own internal metrics, allowing you to collect, process, + and route Vector's internal metrics just like other metrics. + """ classes: { commonly_used: true From 3c241d02823b9c1f28389280bc432f2be2df07a9 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 11:04:08 -0400 Subject: [PATCH 11/43] chore(metrics): add type to generated metric descriptions, restore stripped tag descriptions --- .../commands/build/component_docs/runner.rs | 25 ++- .../components/sources/internal_metrics.cue | 172 ++++-------------- 2 files changed, 50 insertions(+), 147 deletions(-) diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index 6635dfce56bd0..0a0c1fd1e5180 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -364,6 +364,7 @@ fn render_and_import_generated_top_level_config_schema( struct MetricEntry { name: String, + metric_type: &'static str, description: String, deprecated: bool, deprecated_message: Option, @@ -380,14 +381,12 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { let mut entries: Vec = Vec::new(); - for schema in [ - metric_schemas.get("counters"), - metric_schemas.get("histograms"), - metric_schemas.get("gauges"), - ] - .into_iter() - .flatten() - { + for (metric_type, schema) in [ + ("counter", metric_schemas.get("counters")), + ("histogram", metric_schemas.get("histograms")), + ("gauge", metric_schemas.get("gauges")), + ] { + let Some(schema) = schema else { continue }; for variant in schema .get("oneOf") .and_then(Value::as_array) @@ -412,6 +411,7 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { entries.push(MetricEntry { name: name.to_owned(), + metric_type, description: desc.replace('\\', "\\\\").replace('"', "\\\""), deprecated, deprecated_message, @@ -433,18 +433,17 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { ); for e in &entries { + writeln!(cue, "\t{}: {{", e.name).unwrap(); + writeln!(cue, "\t\tdescription: \"{}\"", e.description).unwrap(); + writeln!(cue, "\t\ttype: \"{}\"", e.metric_type).unwrap(); if e.deprecated { - writeln!(cue, "\t{}: {{", e.name).unwrap(); - writeln!(cue, "\t\tdescription: \"{}\"", e.description).unwrap(); cue.push_str("\t\tdeprecated: true\n"); if let Some(msg) = &e.deprecated_message { let escaped = msg.replace('\\', "\\\\").replace('"', "\\\""); writeln!(cue, "\t\tdeprecated_message: \"{escaped}\"").unwrap(); } - cue.push_str("\t}\n"); - } else { - writeln!(cue, "\t{}: description: \"{}\"", e.name, e.description).unwrap(); } + cue.push_str("\t}\n"); } cue.push_str("}\n"); diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index bb4cdfcce20d3..718c265f7eafb 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -42,10 +42,12 @@ components: sources: internal_metrics: { // Default internal metrics tags _internal_metrics_tags: { pid: { + description: "The process ID of the Vector instance." required: false examples: ["4232"] } host: { + description: "The hostname of the system Vector is running on." required: false examples: [_values.local_host] } @@ -53,72 +55,58 @@ components: sources: internal_metrics: { // Instance-level "process" metrics active_clients: { - type: "gauge" default_namespace: "vector" tags: _component_tags } aggregate_events_recorded_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } aggregate_failed_updates: { - type: "counter" default_namespace: "vector" tags: _component_tags } aggregate_flushes_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } api_started_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } component_timed_out_events_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } component_timed_out_requests_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } connection_established_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } connection_send_errors_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } connection_shutdown_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } quit_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } reloaded_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } started_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } stopped_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } @@ -126,59 +114,48 @@ components: sources: internal_metrics: { // Metrics emitted by one or more components // Reusable metric definitions adaptive_concurrency_averaged_rtt: { - type: "histogram" default_namespace: "vector" tags: _component_tags } adaptive_concurrency_in_flight: { - type: "histogram" default_namespace: "vector" tags: _component_tags } adaptive_concurrency_limit: { - type: "histogram" default_namespace: "vector" tags: _component_tags } adaptive_concurrency_observed_rtt: { - type: "histogram" default_namespace: "vector" tags: _component_tags } checkpoints_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } checksum_errors_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } collect_completed_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } collect_duration_seconds: { - type: "histogram" default_namespace: "vector" tags: _internal_metrics_tags } command_executed_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } command_execution_duration_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags } connection_read_errors_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { mode: { @@ -191,123 +168,101 @@ components: sources: internal_metrics: { } } container_processed_events_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } containers_unwatched_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } containers_watched_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } doris_bytes_loaded_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } doris_rows_filtered_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } doris_rows_loaded_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } k8s_format_picker_edge_cases_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } k8s_docker_format_parse_failures_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } events_discarded_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { reason: _reason } } component_latency_seconds: { - type: "histogram" default_namespace: "vector" tags: _internal_metrics_tags } component_latency_mean_seconds: { - type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags } buffer_byte_size: { - type: "gauge" default_namespace: "vector" tags: _component_tags } buffer_events: { - type: "gauge" default_namespace: "vector" tags: _component_tags } buffer_size_bytes: { - type: "gauge" default_namespace: "vector" tags: _component_tags } buffer_size_events: { - type: "gauge" default_namespace: "vector" tags: _component_tags } buffer_discarded_events_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } buffer_received_event_bytes_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } buffer_received_events_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } buffer_send_duration_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags } buffer_sent_event_bytes_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } buffer_sent_events_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } component_discarded_events_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { intentional: { + description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error." required: true } } } component_errors_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { error_type: _error_type @@ -315,205 +270,196 @@ components: sources: internal_metrics: { } } component_received_bytes_total: { - type: "counter" default_namespace: "vector" tags: component_received_events_total.tags } component_received_bytes: { - type: "histogram" default_namespace: "vector" tags: component_received_events_total.tags } component_received_events_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { file: { + description: "The file from which the data originated." required: false } uri: { + description: "The sanitized URI from which the data originated." required: false } container_name: { + description: "The name of the container from which the data originated." required: false } pod_name: { + description: "The name of the pod from which the data originated." required: false } peer_addr: { + description: "The IP from which the data originated." required: false } peer_path: { + description: "The pathname from which the data originated." required: false } mode: _mode } } component_received_events_count: { - type: "histogram" default_namespace: "vector" tags: _component_tags & { file: { + description: "The file from which the data originated." required: false } uri: { + description: "The sanitized URI from which the data originated." required: false } container_name: { + description: "The name of the container from which the data originated." required: false } pod_name: { + description: "The name of the pod from which the data originated." required: false } peer_addr: { + description: "The IP from which the data originated." required: false } peer_path: { + description: "The pathname from which the data originated." required: false } mode: _mode } } component_received_event_bytes_total: { - type: "counter" default_namespace: "vector" tags: component_received_events_total.tags } component_sent_bytes_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { endpoint: { + description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string." required: false } file: { + description: "The absolute path of the destination file." required: false } protocol: { + description: "The protocol used to send the bytes." required: true } region: { + description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname." required: false } } } component_sent_events_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & {output: _output} } component_sent_event_bytes_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & {output: _output} } internal_metrics_cardinality: { - type: "gauge" default_namespace: "vector" tags: {} } internal_metrics_cardinality_total: { - type: "counter" default_namespace: "vector" tags: internal_metrics_cardinality.tags } kafka_queue_messages: { - type: "gauge" default_namespace: "vector" tags: _component_tags } kafka_queue_messages_bytes: { - type: "gauge" default_namespace: "vector" tags: _component_tags } kafka_requests_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } kafka_requests_bytes_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } kafka_responses_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } kafka_responses_bytes_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } kafka_produced_messages_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } kafka_produced_messages_bytes_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } kafka_consumed_messages_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } kafka_consumed_messages_bytes_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } kafka_consumer_lag: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { topic_id: { + description: "The Kafka topic id." required: true } partition_id: { + description: "The Kafka partition id." required: true } } } files_added_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } files_deleted_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } files_resumed_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } files_unwatched_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } open_files: { - type: "counter" default_namespace: "vector" tags: _component_tags } grpc_server_messages_received_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { grpc_method: _grpc_method @@ -521,7 +467,6 @@ components: sources: internal_metrics: { } } grpc_server_messages_sent_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { grpc_method: _grpc_method @@ -530,7 +475,6 @@ components: sources: internal_metrics: { } } grpc_server_handler_duration_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags & { grpc_method: _grpc_method @@ -539,38 +483,32 @@ components: sources: internal_metrics: { } } http_client_response_rtt_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags & { status: _status } } http_client_requests_sent_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { method: _method } } http_client_responses_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { status: _status } } http_client_rtt_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags } http_requests_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } http_server_requests_received_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { method: _method @@ -578,7 +516,6 @@ components: sources: internal_metrics: { } } http_server_responses_sent_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { method: _method @@ -587,7 +524,6 @@ components: sources: internal_metrics: { } } http_server_handler_duration_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags & { method: _method @@ -596,158 +532,135 @@ components: sources: internal_metrics: { } } invalid_record_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } lua_memory_used_bytes: { - type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags } metadata_refresh_failed_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } metadata_refresh_successful_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } open_connections: { - type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags } protobuf_decode_errors_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } send_errors_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } source_lag_time_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags } source_send_batch_latency_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags } source_send_latency_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags } source_buffer_max_byte_size: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_max_event_size: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_max_size_bytes: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_max_size_events: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_utilization: { - type: "histogram" default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_utilization_level: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_utilization_mean: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } splunk_pending_acks: { - type: "gauge" default_namespace: "vector" tags: _component_tags } streams_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } s3_object_processing_failed_duration_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags & { bucket: { + description: "The name of the S3 bucket." required: true } } } s3_object_processing_succeeded_duration_seconds: { - type: "histogram" default_namespace: "vector" tags: _component_tags & { bucket: { + description: "The name of the S3 bucket." required: true } } } sqs_message_delete_succeeded_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } sqs_message_processing_succeeded_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } sqs_message_receive_succeeded_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } sqs_message_received_messages_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } sqs_s3_event_record_ignored_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { ignore_type: { + description: "The reason for ignoring the S3 record" required: true enum: { "invalid_event_kind": "The kind of invalid event." @@ -756,91 +669,87 @@ components: sources: internal_metrics: { } } stale_events_flushed_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } stdin_reads_failed_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } tag_value_limit_exceeded_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { metric_name: { + description: """ + The name of the metric whose tag value limit was exceeded. + Only present when `internal_metrics.include_extended_tags` is enabled. + """ required: false } tag_key: { + description: """ + The key of the tag whose value limit was exceeded. + Only present when `internal_metrics.include_extended_tags` is enabled. + """ required: false } } } timestamp_parse_errors_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } transform_buffer_max_byte_size: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_max_event_size: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_max_size_bytes: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_max_size_events: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_utilization: { - type: "histogram" default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_utilization_level: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_utilization_mean: { - type: "gauge" default_namespace: "vector" tags: _component_tags & { output: _output } } uptime_seconds: { - type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags } utf8_convert_errors_total: { - type: "counter" default_namespace: "vector" tags: _component_tags & { mode: { + description: "The connection mode used by the component." required: true enum: { udp: "User Datagram Protocol" @@ -849,74 +758,69 @@ components: sources: internal_metrics: { } } utilization: { - type: "gauge" default_namespace: "vector" tags: _component_tags } build_info: { - type: "gauge" default_namespace: "vector" tags: _internal_metrics_tags & { debug: { + description: "Whether this is a debug build of Vector" required: true } version: { + description: "Vector version." required: true } rust_version: { + description: "The Rust version from the package manifest." required: true } arch: { + description: "The target architecture being compiled for. (e.g. x86_64)" required: true } revision: { + description: "Revision identifer, related to versioned releases." required: true } } } value_limit_reached_total: { - type: "counter" default_namespace: "vector" tags: _component_tags } // Windows metrics windows_service_install_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } windows_service_restart_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } windows_service_start_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } windows_service_stop_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } windows_service_uninstall_total: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } // config metrics config_reload_rejected: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { reason: _reason } } config_reloaded: { - type: "counter" default_namespace: "vector" tags: _internal_metrics_tags } From 238eced06a76c8564b2918e25eb155682e6281d4 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 11:05:58 -0400 Subject: [PATCH 12/43] chore(metrics): add default_namespace to generated metric descriptions --- .../commands/build/component_docs/runner.rs | 5 +- .../components/sources/internal_metrics.cue | 134 ------------------ 2 files changed, 3 insertions(+), 136 deletions(-) diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index 0a0c1fd1e5180..d747a9dc1c6a5 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -434,8 +434,9 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { for e in &entries { writeln!(cue, "\t{}: {{", e.name).unwrap(); - writeln!(cue, "\t\tdescription: \"{}\"", e.description).unwrap(); - writeln!(cue, "\t\ttype: \"{}\"", e.metric_type).unwrap(); + writeln!(cue, "\t\tdescription: \"{}\"", e.description).unwrap(); + writeln!(cue, "\t\ttype: \"{}\"", e.metric_type).unwrap(); + cue.push_str("\t\tdefault_namespace: \"vector\"\n"); if e.deprecated { cue.push_str("\t\tdeprecated: true\n"); if let Some(msg) = &e.deprecated_message { diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index 718c265f7eafb..fd62eadee26e2 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -55,108 +55,83 @@ components: sources: internal_metrics: { // Instance-level "process" metrics active_clients: { - default_namespace: "vector" tags: _component_tags } aggregate_events_recorded_total: { - default_namespace: "vector" tags: _component_tags } aggregate_failed_updates: { - default_namespace: "vector" tags: _component_tags } aggregate_flushes_total: { - default_namespace: "vector" tags: _component_tags } api_started_total: { - default_namespace: "vector" tags: _internal_metrics_tags } component_timed_out_events_total: { - default_namespace: "vector" tags: _component_tags } component_timed_out_requests_total: { - default_namespace: "vector" tags: _component_tags } connection_established_total: { - default_namespace: "vector" tags: _internal_metrics_tags } connection_send_errors_total: { - default_namespace: "vector" tags: _internal_metrics_tags } connection_shutdown_total: { - default_namespace: "vector" tags: _internal_metrics_tags } quit_total: { - default_namespace: "vector" tags: _internal_metrics_tags } reloaded_total: { - default_namespace: "vector" tags: _internal_metrics_tags } started_total: { - default_namespace: "vector" tags: _internal_metrics_tags } stopped_total: { - default_namespace: "vector" tags: _internal_metrics_tags } // Metrics emitted by one or more components // Reusable metric definitions adaptive_concurrency_averaged_rtt: { - default_namespace: "vector" tags: _component_tags } adaptive_concurrency_in_flight: { - default_namespace: "vector" tags: _component_tags } adaptive_concurrency_limit: { - default_namespace: "vector" tags: _component_tags } adaptive_concurrency_observed_rtt: { - default_namespace: "vector" tags: _component_tags } checkpoints_total: { - default_namespace: "vector" tags: _internal_metrics_tags } checksum_errors_total: { - default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } collect_completed_total: { - default_namespace: "vector" tags: _internal_metrics_tags } collect_duration_seconds: { - default_namespace: "vector" tags: _internal_metrics_tags } command_executed_total: { - default_namespace: "vector" tags: _component_tags } command_execution_duration_seconds: { - default_namespace: "vector" tags: _component_tags } connection_read_errors_total: { - default_namespace: "vector" tags: _component_tags & { mode: { description: "" @@ -168,93 +143,71 @@ components: sources: internal_metrics: { } } container_processed_events_total: { - default_namespace: "vector" tags: _component_tags } containers_unwatched_total: { - default_namespace: "vector" tags: _component_tags } containers_watched_total: { - default_namespace: "vector" tags: _component_tags } doris_bytes_loaded_total: { - default_namespace: "vector" tags: _component_tags } doris_rows_filtered_total: { - default_namespace: "vector" tags: _component_tags } doris_rows_loaded_total: { - default_namespace: "vector" tags: _component_tags } k8s_format_picker_edge_cases_total: { - default_namespace: "vector" tags: _component_tags } k8s_docker_format_parse_failures_total: { - default_namespace: "vector" tags: _component_tags } events_discarded_total: { - default_namespace: "vector" tags: _internal_metrics_tags & { reason: _reason } } component_latency_seconds: { - default_namespace: "vector" tags: _internal_metrics_tags } component_latency_mean_seconds: { - default_namespace: "vector" tags: _internal_metrics_tags } buffer_byte_size: { - default_namespace: "vector" tags: _component_tags } buffer_events: { - default_namespace: "vector" tags: _component_tags } buffer_size_bytes: { - default_namespace: "vector" tags: _component_tags } buffer_size_events: { - default_namespace: "vector" tags: _component_tags } buffer_discarded_events_total: { - default_namespace: "vector" tags: _component_tags } buffer_received_event_bytes_total: { - default_namespace: "vector" tags: _component_tags } buffer_received_events_total: { - default_namespace: "vector" tags: _component_tags } buffer_send_duration_seconds: { - default_namespace: "vector" tags: _component_tags } buffer_sent_event_bytes_total: { - default_namespace: "vector" tags: _component_tags } buffer_sent_events_total: { - default_namespace: "vector" tags: _component_tags } component_discarded_events_total: { - default_namespace: "vector" tags: _component_tags & { intentional: { description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error." @@ -263,22 +216,18 @@ components: sources: internal_metrics: { } } component_errors_total: { - default_namespace: "vector" tags: _component_tags & { error_type: _error_type stage: _stage } } component_received_bytes_total: { - default_namespace: "vector" tags: component_received_events_total.tags } component_received_bytes: { - default_namespace: "vector" tags: component_received_events_total.tags } component_received_events_total: { - default_namespace: "vector" tags: _component_tags & { file: { description: "The file from which the data originated." @@ -308,7 +257,6 @@ components: sources: internal_metrics: { } } component_received_events_count: { - default_namespace: "vector" tags: _component_tags & { file: { description: "The file from which the data originated." @@ -338,11 +286,9 @@ components: sources: internal_metrics: { } } component_received_event_bytes_total: { - default_namespace: "vector" tags: component_received_events_total.tags } component_sent_bytes_total: { - default_namespace: "vector" tags: _component_tags & { endpoint: { description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string." @@ -363,63 +309,48 @@ components: sources: internal_metrics: { } } component_sent_events_total: { - default_namespace: "vector" tags: _component_tags & {output: _output} } component_sent_event_bytes_total: { - default_namespace: "vector" tags: _component_tags & {output: _output} } internal_metrics_cardinality: { - default_namespace: "vector" tags: {} } internal_metrics_cardinality_total: { - default_namespace: "vector" tags: internal_metrics_cardinality.tags } kafka_queue_messages: { - default_namespace: "vector" tags: _component_tags } kafka_queue_messages_bytes: { - default_namespace: "vector" tags: _component_tags } kafka_requests_total: { - default_namespace: "vector" tags: _component_tags } kafka_requests_bytes_total: { - default_namespace: "vector" tags: _component_tags } kafka_responses_total: { - default_namespace: "vector" tags: _component_tags } kafka_responses_bytes_total: { - default_namespace: "vector" tags: _component_tags } kafka_produced_messages_total: { - default_namespace: "vector" tags: _component_tags } kafka_produced_messages_bytes_total: { - default_namespace: "vector" tags: _component_tags } kafka_consumed_messages_total: { - default_namespace: "vector" tags: _component_tags } kafka_consumed_messages_bytes_total: { - default_namespace: "vector" tags: _component_tags } kafka_consumer_lag: { - default_namespace: "vector" tags: _component_tags & { topic_id: { description: "The Kafka topic id." @@ -432,42 +363,35 @@ components: sources: internal_metrics: { } } files_added_total: { - default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } files_deleted_total: { - default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } files_resumed_total: { - default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } files_unwatched_total: { - default_namespace: "vector" tags: _internal_metrics_tags & { file: _file } } open_files: { - default_namespace: "vector" tags: _component_tags } grpc_server_messages_received_total: { - default_namespace: "vector" tags: _component_tags & { grpc_method: _grpc_method grpc_service: _grpc_service } } grpc_server_messages_sent_total: { - default_namespace: "vector" tags: _component_tags & { grpc_method: _grpc_method grpc_service: _grpc_service @@ -475,7 +399,6 @@ components: sources: internal_metrics: { } } grpc_server_handler_duration_seconds: { - default_namespace: "vector" tags: _component_tags & { grpc_method: _grpc_method grpc_service: _grpc_service @@ -483,40 +406,33 @@ components: sources: internal_metrics: { } } http_client_response_rtt_seconds: { - default_namespace: "vector" tags: _component_tags & { status: _status } } http_client_requests_sent_total: { - default_namespace: "vector" tags: _component_tags & { method: _method } } http_client_responses_total: { - default_namespace: "vector" tags: _component_tags & { status: _status } } http_client_rtt_seconds: { - default_namespace: "vector" tags: _component_tags } http_requests_total: { - default_namespace: "vector" tags: _component_tags } http_server_requests_received_total: { - default_namespace: "vector" tags: _component_tags & { method: _method path: _path } } http_server_responses_sent_total: { - default_namespace: "vector" tags: _component_tags & { method: _method path: _path @@ -524,7 +440,6 @@ components: sources: internal_metrics: { } } http_server_handler_duration_seconds: { - default_namespace: "vector" tags: _component_tags & { method: _method path: _path @@ -532,97 +447,77 @@ components: sources: internal_metrics: { } } invalid_record_total: { - default_namespace: "vector" tags: _component_tags } lua_memory_used_bytes: { - default_namespace: "vector" tags: _internal_metrics_tags } metadata_refresh_failed_total: { - default_namespace: "vector" tags: _component_tags } metadata_refresh_successful_total: { - default_namespace: "vector" tags: _component_tags } open_connections: { - default_namespace: "vector" tags: _internal_metrics_tags } protobuf_decode_errors_total: { - default_namespace: "vector" tags: _component_tags } send_errors_total: { - default_namespace: "vector" tags: _component_tags } source_lag_time_seconds: { - default_namespace: "vector" tags: _component_tags } source_send_batch_latency_seconds: { - default_namespace: "vector" tags: _component_tags } source_send_latency_seconds: { - default_namespace: "vector" tags: _component_tags } source_buffer_max_byte_size: { - default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_max_event_size: { - default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_max_size_bytes: { - default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_max_size_events: { - default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_utilization: { - default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_utilization_level: { - default_namespace: "vector" tags: _component_tags & { output: _output } } source_buffer_utilization_mean: { - default_namespace: "vector" tags: _component_tags & { output: _output } } splunk_pending_acks: { - default_namespace: "vector" tags: _component_tags } streams_total: { - default_namespace: "vector" tags: _component_tags } s3_object_processing_failed_duration_seconds: { - default_namespace: "vector" tags: _component_tags & { bucket: { description: "The name of the S3 bucket." @@ -631,7 +526,6 @@ components: sources: internal_metrics: { } } s3_object_processing_succeeded_duration_seconds: { - default_namespace: "vector" tags: _component_tags & { bucket: { description: "The name of the S3 bucket." @@ -640,23 +534,18 @@ components: sources: internal_metrics: { } } sqs_message_delete_succeeded_total: { - default_namespace: "vector" tags: _component_tags } sqs_message_processing_succeeded_total: { - default_namespace: "vector" tags: _component_tags } sqs_message_receive_succeeded_total: { - default_namespace: "vector" tags: _component_tags } sqs_message_received_messages_total: { - default_namespace: "vector" tags: _component_tags } sqs_s3_event_record_ignored_total: { - default_namespace: "vector" tags: _component_tags & { ignore_type: { @@ -669,15 +558,12 @@ components: sources: internal_metrics: { } } stale_events_flushed_total: { - default_namespace: "vector" tags: _component_tags } stdin_reads_failed_total: { - default_namespace: "vector" tags: _component_tags } tag_value_limit_exceeded_total: { - default_namespace: "vector" tags: _component_tags & { metric_name: { description: """ @@ -696,57 +582,47 @@ components: sources: internal_metrics: { } } timestamp_parse_errors_total: { - default_namespace: "vector" tags: _component_tags } transform_buffer_max_byte_size: { - default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_max_event_size: { - default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_max_size_bytes: { - default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_max_size_events: { - default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_utilization: { - default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_utilization_level: { - default_namespace: "vector" tags: _component_tags & { output: _output } } transform_buffer_utilization_mean: { - default_namespace: "vector" tags: _component_tags & { output: _output } } uptime_seconds: { - default_namespace: "vector" tags: _internal_metrics_tags } utf8_convert_errors_total: { - default_namespace: "vector" tags: _component_tags & { mode: { description: "The connection mode used by the component." @@ -758,11 +634,9 @@ components: sources: internal_metrics: { } } utilization: { - default_namespace: "vector" tags: _component_tags } build_info: { - default_namespace: "vector" tags: _internal_metrics_tags & { debug: { description: "Whether this is a debug build of Vector" @@ -787,41 +661,33 @@ components: sources: internal_metrics: { } } value_limit_reached_total: { - default_namespace: "vector" tags: _component_tags } // Windows metrics windows_service_install_total: { - default_namespace: "vector" tags: _internal_metrics_tags } windows_service_restart_total: { - default_namespace: "vector" tags: _internal_metrics_tags } windows_service_start_total: { - default_namespace: "vector" tags: _internal_metrics_tags } windows_service_stop_total: { - default_namespace: "vector" tags: _internal_metrics_tags } windows_service_uninstall_total: { - default_namespace: "vector" tags: _internal_metrics_tags } // config metrics config_reload_rejected: { - default_namespace: "vector" tags: _internal_metrics_tags & { reason: _reason } } config_reloaded: { - default_namespace: "vector" tags: _internal_metrics_tags } From 76f6b264e92704d04cee069cfdd7f0e86f493a58 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 11:14:02 -0400 Subject: [PATCH 13/43] chore(metrics): add tags to generated metric descriptions --- .../src/internal_event/metric_name.rs | 21 +++++ .../commands/build/component_docs/runner.rs | 9 ++ .../components/sources/internal_metrics.cue | 84 ------------------- 3 files changed, 30 insertions(+), 84 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index 3c81ca848af62..ecb553fb31c18 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -71,27 +71,33 @@ pub enum CounterName { AggregateFlushesTotal, /// The number of times the Vector API has been started. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] ApiStartedTotal, /// The total number of files checkpointed. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] CheckpointsTotal, /// The total number of errors identifying files via checksum. ChecksumErrorsTotal, /// The total number of metrics collections completed for this component. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] CollectCompletedTotal, /// The total number of times a command has been executed. CommandExecutedTotal, /// The total number of times a connection has been established. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] ConnectionEstablishedTotal, /// The total number of errors sending data via the connection. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] ConnectionSendErrorsTotal, /// The total number of times the connection has been shut down. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] ConnectionShutdownTotal, /// The total number of container events processed. @@ -191,9 +197,11 @@ pub enum CounterName { ParseErrorsTotal, /// The total number of times the Vector instance has quit. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] QuitTotal, /// The total number of times the Vector instance has been reloaded. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] ReloadedTotal, /// The total number of events with rewrapped timestamps. @@ -218,9 +226,11 @@ pub enum CounterName { StaleEventsFlushedTotal, /// The total number of times the Vector instance has been started. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] StartedTotal, /// The total number of times the Vector instance has been stopped. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] StoppedTotal, /// The total number of events that contained a tag which exceeded the configured cardinality limit. @@ -239,18 +249,23 @@ pub enum CounterName { WebsocketMessagesSentTotal, /// The total number of times the Windows service has been installed. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] WindowsServiceInstallTotal, /// The total number of times the Windows service has been restarted. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] WindowsServiceRestartTotal, /// The total number of times the Windows service has been started. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] WindowsServiceStartTotal, /// The total number of times the Windows service has been stopped. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] WindowsServiceStopTotal, /// The total number of times the Windows service has been uninstalled. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] WindowsServiceUninstallTotal, /// The total number of failures to annotate Kubernetes events with namespace metadata. @@ -315,6 +330,7 @@ pub enum HistogramName { /// /// This includes both the time spent queued in the transform's input buffer and the time spent /// executing the transform itself. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] ComponentLatencySeconds, /// The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds. @@ -354,6 +370,7 @@ pub enum HistogramName { S3ObjectProcessingFailedDurationSeconds, /// The duration spent collecting metrics for this component. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] CollectDurationSeconds, /// The command execution duration in seconds. @@ -429,6 +446,7 @@ pub enum GaugeName { /// This includes both the time spent queued in the transform's input buffer and the time spent /// executing the transform itself. This value is smoothed over time using an exponentially /// weighted moving average (EWMA). + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] ComponentLatencyMeanSeconds, /// The maximum number of events the source buffer can hold. @@ -507,6 +525,7 @@ pub enum GaugeName { OpenFiles, /// The number of seconds the Vector instance has been running. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] UptimeSeconds, /// Pseudo-metric that provides build information for the Vector instance. @@ -522,9 +541,11 @@ pub enum GaugeName { KafkaConsumerLag, /// The total memory currently being used by the Lua runtime. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] LuaMemoryUsedBytes, /// The number of current open connections to Vector. + #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] OpenConnections, /// The number of currently active endpoints. diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index d747a9dc1c6a5..c3ed78edb3b02 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -366,6 +366,7 @@ struct MetricEntry { name: String, metric_type: &'static str, description: String, + tags: String, deprecated: bool, deprecated_message: Option, } @@ -408,11 +409,18 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { .and_then(|m| m.get("deprecated_message")) .and_then(Value::as_str) .map(str::to_owned); + let tags = variant + .get("_metadata") + .and_then(|m| m.get("docs::tags")) + .and_then(Value::as_str) + .unwrap_or("_component_tags") + .to_owned(); entries.push(MetricEntry { name: name.to_owned(), metric_type, description: desc.replace('\\', "\\\\").replace('"', "\\\""), + tags, deprecated, deprecated_message, }); @@ -437,6 +445,7 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { writeln!(cue, "\t\tdescription: \"{}\"", e.description).unwrap(); writeln!(cue, "\t\ttype: \"{}\"", e.metric_type).unwrap(); cue.push_str("\t\tdefault_namespace: \"vector\"\n"); + writeln!(cue, "\t\ttags: {}", e.tags).unwrap(); if e.deprecated { cue.push_str("\t\tdeprecated: true\n"); if let Some(msg) = &e.deprecated_message { diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index fd62eadee26e2..b55c840385a59 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -55,64 +55,45 @@ components: sources: internal_metrics: { // Instance-level "process" metrics active_clients: { - tags: _component_tags } aggregate_events_recorded_total: { - tags: _component_tags } aggregate_failed_updates: { - tags: _component_tags } aggregate_flushes_total: { - tags: _component_tags } api_started_total: { - tags: _internal_metrics_tags } component_timed_out_events_total: { - tags: _component_tags } component_timed_out_requests_total: { - tags: _component_tags } connection_established_total: { - tags: _internal_metrics_tags } connection_send_errors_total: { - tags: _internal_metrics_tags } connection_shutdown_total: { - tags: _internal_metrics_tags } quit_total: { - tags: _internal_metrics_tags } reloaded_total: { - tags: _internal_metrics_tags } started_total: { - tags: _internal_metrics_tags } stopped_total: { - tags: _internal_metrics_tags } // Metrics emitted by one or more components // Reusable metric definitions adaptive_concurrency_averaged_rtt: { - tags: _component_tags } adaptive_concurrency_in_flight: { - tags: _component_tags } adaptive_concurrency_limit: { - tags: _component_tags } adaptive_concurrency_observed_rtt: { - tags: _component_tags } checkpoints_total: { - tags: _internal_metrics_tags } checksum_errors_total: { tags: _internal_metrics_tags & { @@ -120,16 +101,12 @@ components: sources: internal_metrics: { } } collect_completed_total: { - tags: _internal_metrics_tags } collect_duration_seconds: { - tags: _internal_metrics_tags } command_executed_total: { - tags: _component_tags } command_execution_duration_seconds: { - tags: _component_tags } connection_read_errors_total: { tags: _component_tags & { @@ -143,28 +120,20 @@ components: sources: internal_metrics: { } } container_processed_events_total: { - tags: _component_tags } containers_unwatched_total: { - tags: _component_tags } containers_watched_total: { - tags: _component_tags } doris_bytes_loaded_total: { - tags: _component_tags } doris_rows_filtered_total: { - tags: _component_tags } doris_rows_loaded_total: { - tags: _component_tags } k8s_format_picker_edge_cases_total: { - tags: _component_tags } k8s_docker_format_parse_failures_total: { - tags: _component_tags } events_discarded_total: { tags: _internal_metrics_tags & { @@ -172,40 +141,28 @@ components: sources: internal_metrics: { } } component_latency_seconds: { - tags: _internal_metrics_tags } component_latency_mean_seconds: { - tags: _internal_metrics_tags } buffer_byte_size: { - tags: _component_tags } buffer_events: { - tags: _component_tags } buffer_size_bytes: { - tags: _component_tags } buffer_size_events: { - tags: _component_tags } buffer_discarded_events_total: { - tags: _component_tags } buffer_received_event_bytes_total: { - tags: _component_tags } buffer_received_events_total: { - tags: _component_tags } buffer_send_duration_seconds: { - tags: _component_tags } buffer_sent_event_bytes_total: { - tags: _component_tags } buffer_sent_events_total: { - tags: _component_tags } component_discarded_events_total: { tags: _component_tags & { @@ -321,34 +278,24 @@ components: sources: internal_metrics: { tags: internal_metrics_cardinality.tags } kafka_queue_messages: { - tags: _component_tags } kafka_queue_messages_bytes: { - tags: _component_tags } kafka_requests_total: { - tags: _component_tags } kafka_requests_bytes_total: { - tags: _component_tags } kafka_responses_total: { - tags: _component_tags } kafka_responses_bytes_total: { - tags: _component_tags } kafka_produced_messages_total: { - tags: _component_tags } kafka_produced_messages_bytes_total: { - tags: _component_tags } kafka_consumed_messages_total: { - tags: _component_tags } kafka_consumed_messages_bytes_total: { - tags: _component_tags } kafka_consumer_lag: { tags: _component_tags & { @@ -383,7 +330,6 @@ components: sources: internal_metrics: { } } open_files: { - tags: _component_tags } grpc_server_messages_received_total: { tags: _component_tags & { @@ -421,10 +367,8 @@ components: sources: internal_metrics: { } } http_client_rtt_seconds: { - tags: _component_tags } http_requests_total: { - tags: _component_tags } http_server_requests_received_total: { tags: _component_tags & { @@ -447,34 +391,24 @@ components: sources: internal_metrics: { } } invalid_record_total: { - tags: _component_tags } lua_memory_used_bytes: { - tags: _internal_metrics_tags } metadata_refresh_failed_total: { - tags: _component_tags } metadata_refresh_successful_total: { - tags: _component_tags } open_connections: { - tags: _internal_metrics_tags } protobuf_decode_errors_total: { - tags: _component_tags } send_errors_total: { - tags: _component_tags } source_lag_time_seconds: { - tags: _component_tags } source_send_batch_latency_seconds: { - tags: _component_tags } source_send_latency_seconds: { - tags: _component_tags } source_buffer_max_byte_size: { tags: _component_tags & { @@ -512,10 +446,8 @@ components: sources: internal_metrics: { } } splunk_pending_acks: { - tags: _component_tags } streams_total: { - tags: _component_tags } s3_object_processing_failed_duration_seconds: { tags: _component_tags & { @@ -534,16 +466,12 @@ components: sources: internal_metrics: { } } sqs_message_delete_succeeded_total: { - tags: _component_tags } sqs_message_processing_succeeded_total: { - tags: _component_tags } sqs_message_receive_succeeded_total: { - tags: _component_tags } sqs_message_received_messages_total: { - tags: _component_tags } sqs_s3_event_record_ignored_total: { @@ -558,10 +486,8 @@ components: sources: internal_metrics: { } } stale_events_flushed_total: { - tags: _component_tags } stdin_reads_failed_total: { - tags: _component_tags } tag_value_limit_exceeded_total: { tags: _component_tags & { @@ -582,7 +508,6 @@ components: sources: internal_metrics: { } } timestamp_parse_errors_total: { - tags: _component_tags } transform_buffer_max_byte_size: { tags: _component_tags & { @@ -620,7 +545,6 @@ components: sources: internal_metrics: { } } uptime_seconds: { - tags: _internal_metrics_tags } utf8_convert_errors_total: { tags: _component_tags & { @@ -634,7 +558,6 @@ components: sources: internal_metrics: { } } utilization: { - tags: _component_tags } build_info: { tags: _internal_metrics_tags & { @@ -661,24 +584,18 @@ components: sources: internal_metrics: { } } value_limit_reached_total: { - tags: _component_tags } // Windows metrics windows_service_install_total: { - tags: _internal_metrics_tags } windows_service_restart_total: { - tags: _internal_metrics_tags } windows_service_start_total: { - tags: _internal_metrics_tags } windows_service_stop_total: { - tags: _internal_metrics_tags } windows_service_uninstall_total: { - tags: _internal_metrics_tags } // config metrics @@ -688,7 +605,6 @@ components: sources: internal_metrics: { } } config_reloaded: { - tags: _internal_metrics_tags } // Helpful tag groupings From 35bea483a796f8973bb3f78946366fda4fa48815 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 11:17:11 -0400 Subject: [PATCH 14/43] fix(docs): remove empty metric blocks from internal_metrics.cue --- .../components/sources/internal_metrics.cue | 168 ------------------ 1 file changed, 168 deletions(-) diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index b55c840385a59..edf76fdbd9759 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -54,60 +54,14 @@ components: sources: internal_metrics: { } // Instance-level "process" metrics - active_clients: { - } - aggregate_events_recorded_total: { - } - aggregate_failed_updates: { - } - aggregate_flushes_total: { - } - api_started_total: { - } - component_timed_out_events_total: { - } - component_timed_out_requests_total: { - } - connection_established_total: { - } - connection_send_errors_total: { - } - connection_shutdown_total: { - } - quit_total: { - } - reloaded_total: { - } - started_total: { - } - stopped_total: { - } // Metrics emitted by one or more components // Reusable metric definitions - adaptive_concurrency_averaged_rtt: { - } - adaptive_concurrency_in_flight: { - } - adaptive_concurrency_limit: { - } - adaptive_concurrency_observed_rtt: { - } - checkpoints_total: { - } checksum_errors_total: { tags: _internal_metrics_tags & { file: _file } } - collect_completed_total: { - } - collect_duration_seconds: { - } - command_executed_total: { - } - command_execution_duration_seconds: { - } connection_read_errors_total: { tags: _component_tags & { mode: { @@ -119,51 +73,11 @@ components: sources: internal_metrics: { } } } - container_processed_events_total: { - } - containers_unwatched_total: { - } - containers_watched_total: { - } - doris_bytes_loaded_total: { - } - doris_rows_filtered_total: { - } - doris_rows_loaded_total: { - } - k8s_format_picker_edge_cases_total: { - } - k8s_docker_format_parse_failures_total: { - } events_discarded_total: { tags: _internal_metrics_tags & { reason: _reason } } - component_latency_seconds: { - } - component_latency_mean_seconds: { - } - buffer_byte_size: { - } - buffer_events: { - } - buffer_size_bytes: { - } - buffer_size_events: { - } - buffer_discarded_events_total: { - } - buffer_received_event_bytes_total: { - } - buffer_received_events_total: { - } - buffer_send_duration_seconds: { - } - buffer_sent_event_bytes_total: { - } - buffer_sent_events_total: { - } component_discarded_events_total: { tags: _component_tags & { intentional: { @@ -277,26 +191,6 @@ components: sources: internal_metrics: { internal_metrics_cardinality_total: { tags: internal_metrics_cardinality.tags } - kafka_queue_messages: { - } - kafka_queue_messages_bytes: { - } - kafka_requests_total: { - } - kafka_requests_bytes_total: { - } - kafka_responses_total: { - } - kafka_responses_bytes_total: { - } - kafka_produced_messages_total: { - } - kafka_produced_messages_bytes_total: { - } - kafka_consumed_messages_total: { - } - kafka_consumed_messages_bytes_total: { - } kafka_consumer_lag: { tags: _component_tags & { topic_id: { @@ -329,8 +223,6 @@ components: sources: internal_metrics: { file: _file } } - open_files: { - } grpc_server_messages_received_total: { tags: _component_tags & { grpc_method: _grpc_method @@ -366,10 +258,6 @@ components: sources: internal_metrics: { status: _status } } - http_client_rtt_seconds: { - } - http_requests_total: { - } http_server_requests_received_total: { tags: _component_tags & { method: _method @@ -390,26 +278,6 @@ components: sources: internal_metrics: { status: _status } } - invalid_record_total: { - } - lua_memory_used_bytes: { - } - metadata_refresh_failed_total: { - } - metadata_refresh_successful_total: { - } - open_connections: { - } - protobuf_decode_errors_total: { - } - send_errors_total: { - } - source_lag_time_seconds: { - } - source_send_batch_latency_seconds: { - } - source_send_latency_seconds: { - } source_buffer_max_byte_size: { tags: _component_tags & { output: _output @@ -445,10 +313,6 @@ components: sources: internal_metrics: { output: _output } } - splunk_pending_acks: { - } - streams_total: { - } s3_object_processing_failed_duration_seconds: { tags: _component_tags & { bucket: { @@ -465,14 +329,6 @@ components: sources: internal_metrics: { } } } - sqs_message_delete_succeeded_total: { - } - sqs_message_processing_succeeded_total: { - } - sqs_message_receive_succeeded_total: { - } - sqs_message_received_messages_total: { - } sqs_s3_event_record_ignored_total: { tags: _component_tags & { @@ -485,10 +341,6 @@ components: sources: internal_metrics: { } } } - stale_events_flushed_total: { - } - stdin_reads_failed_total: { - } tag_value_limit_exceeded_total: { tags: _component_tags & { metric_name: { @@ -507,8 +359,6 @@ components: sources: internal_metrics: { } } } - timestamp_parse_errors_total: { - } transform_buffer_max_byte_size: { tags: _component_tags & { output: _output @@ -544,8 +394,6 @@ components: sources: internal_metrics: { output: _output } } - uptime_seconds: { - } utf8_convert_errors_total: { tags: _component_tags & { mode: { @@ -557,8 +405,6 @@ components: sources: internal_metrics: { } } } - utilization: { - } build_info: { tags: _internal_metrics_tags & { debug: { @@ -583,20 +429,8 @@ components: sources: internal_metrics: { } } } - value_limit_reached_total: { - } // Windows metrics - windows_service_install_total: { - } - windows_service_restart_total: { - } - windows_service_start_total: { - } - windows_service_stop_total: { - } - windows_service_uninstall_total: { - } // config metrics config_reload_rejected: { @@ -604,8 +438,6 @@ components: sources: internal_metrics: { reason: _reason } } - config_reloaded: { - } // Helpful tag groupings _component_tags: _internal_metrics_tags & { From 7291028312d53f94c5d7e6f150151906ae001526 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 11:58:39 -0400 Subject: [PATCH 15/43] chore(metrics): use Rust constants for tag sets, encode all complex tags in Rust, eliminate enum metric blocks from internal_metrics.cue --- .../src/internal_event/metric_name.rs | 104 ++++- lib/vector-config-macros/src/ast/mod.rs | 6 + .../components/sources/internal_metrics.cue | 393 ++---------------- 3 files changed, 122 insertions(+), 381 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index ecb553fb31c18..b94488d72decf 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -1,6 +1,23 @@ use strum::{AsRefStr, Display, EnumIter}; use vector_config::configurable_component; +// CUE tag-set constants referenced via `#[configurable(metadata(docs::tags = CONSTANT))]`. +// The values are raw CUE expressions emitted verbatim by `vdev build component-docs`. +pub const COMPONENT_TAGS: &str = "_component_tags"; +pub const INTERNAL_METRICS_TAGS: &str = "_internal_metrics_tags"; +pub const COMPONENT_TAGS_OUTPUT: &str = "_component_tags & {output: _output}"; +pub const INTERNAL_METRICS_TAGS_FILE: &str = "_internal_metrics_tags & {file: _file}"; +pub const INTERNAL_METRICS_TAGS_REASON: &str = "_internal_metrics_tags & {reason: _reason}"; +pub const COMPONENT_TAGS_GRPC_METHOD_SERVICE: &str = + "_component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service}"; +pub const COMPONENT_TAGS_GRPC_ALL: &str = "_component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service, grpc_status: _grpc_status}"; +pub const COMPONENT_TAGS_HTTP_METHOD: &str = "_component_tags & {method: _method}"; +pub const COMPONENT_TAGS_HTTP_STATUS: &str = "_component_tags & {status: _status}"; +pub const COMPONENT_TAGS_HTTP_METHOD_PATH: &str = "_component_tags & {method: _method, path: _path}"; +pub const COMPONENT_TAGS_HTTP_ALL: &str = + "_component_tags & {method: _method, path: _path, status: _status}"; +pub const COMPONENT_RECEIVED_EVENTS_TAGS: &str = "component_received_events_total.tags"; + /// Canonical list of all per-component internal counter metric names emitted by Vector. #[configurable_component] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] @@ -9,28 +26,36 @@ use vector_config::configurable_component; pub enum CounterName { /// The number of events accepted by this component either from tagged /// origins like file and uri, or cumulatively from other origins. + #[configurable(metadata(docs::tags = "_component_tags & {file: {description: \"The file from which the data originated.\", required: false}, uri: {description: \"The sanitized URI from which the data originated.\", required: false}, container_name: {description: \"The name of the container from which the data originated.\", required: false}, pod_name: {description: \"The name of the pod from which the data originated.\", required: false}, peer_addr: {description: \"The IP from which the data originated.\", required: false}, peer_path: {description: \"The pathname from which the data originated.\", required: false}, mode: _mode}"))] ComponentReceivedEventsTotal, /// The number of event bytes accepted by this component either from /// tagged origins like file and uri, or cumulatively from other origins. + #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TAGS))] ComponentReceivedEventBytesTotal, /// The number of raw bytes accepted by this component from source origins. + #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TAGS))] ComponentReceivedBytesTotal, /// The total number of events emitted by this component. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] ComponentSentEventsTotal, /// The total number of event bytes emitted by this component. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] ComponentSentEventBytesTotal, /// The number of raw bytes sent by this component to destination sinks. + #[configurable(metadata(docs::tags = "_component_tags & {endpoint: {description: \"The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.\", required: false}, file: {description: \"The absolute path of the destination file.\", required: false}, protocol: {description: \"The protocol used to send the bytes.\", required: true}, region: {description: \"The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.\", required: false}}"))] ComponentSentBytesTotal, /// The number of events dropped by this component. + #[configurable(metadata(docs::tags = "_component_tags & {intentional: {description: \"True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.\", required: true}}"))] ComponentDiscardedEventsTotal, /// The total number of errors encountered by this component. + #[configurable(metadata(docs::tags = "_component_tags & {error_type: _error_type, stage: _stage}"))] ComponentErrorsTotal, /// The total number of events for which this source responded with a timeout error. @@ -71,33 +96,34 @@ pub enum CounterName { AggregateFlushesTotal, /// The number of times the Vector API has been started. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] ApiStartedTotal, /// The total number of files checkpointed. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] CheckpointsTotal, /// The total number of errors identifying files via checksum. + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] ChecksumErrorsTotal, /// The total number of metrics collections completed for this component. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] CollectCompletedTotal, /// The total number of times a command has been executed. CommandExecutedTotal, /// The total number of times a connection has been established. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] ConnectionEstablishedTotal, /// The total number of errors sending data via the connection. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] ConnectionSendErrorsTotal, /// The total number of times the connection has been shut down. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] ConnectionShutdownTotal, /// The total number of container events processed. @@ -128,39 +154,50 @@ pub enum CounterName { EncoderUnmappableReplacementWarningsTotal, /// The total number of events discarded by this component. + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_REASON))] EventsDiscardedTotal, /// The total number of files Vector has found to watch. + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] FilesAddedTotal, /// The total number of files deleted. + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] FilesDeletedTotal, /// The total number of times Vector has resumed watching a file. + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] FilesResumedTotal, /// The total number of times Vector has stopped watching a file. + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] FilesUnwatchedTotal, /// The total number of gRPC messages received. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_GRPC_METHOD_SERVICE))] GrpcServerMessagesReceivedTotal, /// The total number of gRPC messages sent. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_GRPC_ALL))] GrpcServerMessagesSentTotal, /// The total number of HTTP client errors encountered. HttpClientErrorsTotal, /// The total number of sent HTTP requests, tagged with the request method. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_METHOD))] HttpClientRequestsSentTotal, /// The total number of HTTP requests, tagged with the response code. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_STATUS))] HttpClientResponsesTotal, /// The total number of HTTP requests received. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_METHOD_PATH))] HttpServerRequestsReceivedTotal, /// The total number of HTTP responses sent. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_ALL))] HttpServerResponsesSentTotal, /// Total number of message bytes (including framing) received from Kafka brokers. @@ -197,11 +234,11 @@ pub enum CounterName { ParseErrorsTotal, /// The total number of times the Vector instance has quit. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] QuitTotal, /// The total number of times the Vector instance has been reloaded. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] ReloadedTotal, /// The total number of events with rewrapped timestamps. @@ -226,17 +263,18 @@ pub enum CounterName { StaleEventsFlushedTotal, /// The total number of times the Vector instance has been started. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] StartedTotal, /// The total number of times the Vector instance has been stopped. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] StoppedTotal, /// The total number of events that contained a tag which exceeded the configured cardinality limit. TagCardinalityUntrackedEventsTotal, /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. + #[configurable(metadata(docs::tags = "_component_tags & {metric_name: {description: \"The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.\", required: false}, tag_key: {description: \"The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.\", required: false}}"))] TagValueLimitExceededTotal, /// The total number of times the value limit was reached. @@ -249,23 +287,23 @@ pub enum CounterName { WebsocketMessagesSentTotal, /// The total number of times the Windows service has been installed. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] WindowsServiceInstallTotal, /// The total number of times the Windows service has been restarted. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] WindowsServiceRestartTotal, /// The total number of times the Windows service has been started. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] WindowsServiceStartTotal, /// The total number of times the Windows service has been stopped. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] WindowsServiceStopTotal, /// The total number of times the Windows service has been uninstalled. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] WindowsServiceUninstallTotal, /// The total number of failures to annotate Kubernetes events with namespace metadata. @@ -281,6 +319,7 @@ pub enum CounterName { K8sDockerFormatParseFailuresTotal, /// The total number of times an S3 record in an SQS message was ignored. + #[configurable(metadata(docs::tags = "_component_tags & {ignore_type: {description: \"The reason for ignoring the S3 record\", required: true, enum: {invalid_event_kind: \"The kind of invalid event.\"}}}"))] SqsS3EventRecordIgnoredTotal, /// The total number of bytes allocated by this component. @@ -318,9 +357,11 @@ pub enum HistogramName { /// /// Note that this is separate than sink-level batching. It is mostly useful for low level /// debugging performance issues in Vector due to small internal batches. + #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TAGS))] ComponentReceivedEventsCount, /// The size in bytes of each event received by the source. + #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TAGS))] ComponentReceivedBytes, /// The duration spent sending a payload to this buffer. @@ -330,7 +371,7 @@ pub enum HistogramName { /// /// This includes both the time spent queued in the transform's input buffer and the time spent /// executing the transform itself. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] ComponentLatencySeconds, /// The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds. @@ -364,37 +405,44 @@ pub enum HistogramName { AdaptiveConcurrencyReachedLimit, /// The time taken to process an S3 object that succeeded, in seconds. + #[configurable(metadata(docs::tags = "_component_tags & {bucket: {description: \"The name of the S3 bucket.\", required: true}}"))] S3ObjectProcessingSucceededDurationSeconds, /// The time taken to process an S3 object that failed, in seconds. + #[configurable(metadata(docs::tags = "_component_tags & {bucket: {description: \"The name of the S3 bucket.\", required: true}}"))] S3ObjectProcessingFailedDurationSeconds, /// The duration spent collecting metrics for this component. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] CollectDurationSeconds, /// The command execution duration in seconds. CommandExecutionDurationSeconds, /// The duration spent handling a gRPC request. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_GRPC_ALL))] GrpcServerHandlerDurationSeconds, /// The duration spent handling an HTTP request. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_ALL))] HttpServerHandlerDurationSeconds, /// The round-trip time (RTT) of HTTP requests. HttpClientRttSeconds, /// The round-trip time (RTT) of HTTP requests, tagged with the response code. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_STATUS))] HttpClientResponseRttSeconds, /// The round-trip time (RTT) of HTTP requests that resulted in an error. HttpClientErrorRttSeconds, /// The utilization of the source buffer. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] SourceBufferUtilization, /// The utilization of the buffer that feeds into a transform. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] TransformBufferUtilization, } @@ -446,47 +494,59 @@ pub enum GaugeName { /// This includes both the time spent queued in the transform's input buffer and the time spent /// executing the transform itself. This value is smoothed over time using an exponentially /// weighted moving average (EWMA). - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] ComponentLatencyMeanSeconds, /// The maximum number of events the source buffer can hold. #[configurable(deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_events`.")] + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] SourceBufferMaxEventSize, /// The maximum number of bytes the source buffer can hold. #[configurable(deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_bytes`.")] + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] SourceBufferMaxByteSize, /// The maximum number of events the source buffer can hold. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] SourceBufferMaxSizeEvents, /// The maximum number of bytes the source buffer can hold. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] SourceBufferMaxSizeBytes, /// The current utilization level of the source buffer. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] SourceBufferUtilizationLevel, /// The mean utilization of the source buffer, smoothed with an EWMA. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] SourceBufferUtilizationMean, /// The maximum number of events the buffer that feeds into a transform can hold. #[configurable(deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_events`.")] + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] TransformBufferMaxEventSize, /// The maximum number of bytes the buffer that feeds into a transform can hold. #[configurable(deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_bytes`.")] + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] TransformBufferMaxByteSize, /// The maximum number of events the buffer that feeds into a transform can hold. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] TransformBufferMaxSizeEvents, /// The maximum number of bytes the buffer that feeds into a transform can hold. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] TransformBufferMaxSizeBytes, /// The current utilization level of the buffer that feeds into a transform. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] TransformBufferUtilizationLevel, /// The mean utilization of the buffer that feeds into a transform, smoothed with an EWMA. + #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] TransformBufferUtilizationMean, /// The maximum number of events in the buffer. @@ -525,10 +585,11 @@ pub enum GaugeName { OpenFiles, /// The number of seconds the Vector instance has been running. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] UptimeSeconds, /// Pseudo-metric that provides build information for the Vector instance. + #[configurable(metadata(docs::tags = "_internal_metrics_tags & {debug: {description: \"Whether this is a debug build of Vector\", required: true}, version: {description: \"Vector version.\", required: true}, rust_version: {description: \"The Rust version from the package manifest.\", required: true}, arch: {description: \"The target architecture being compiled for. (e.g. x86_64)\", required: true}, revision: {description: \"Revision identifer, related to versioned releases.\", required: true}}"))] BuildInfo, /// Current number of messages in producer queues. @@ -538,14 +599,15 @@ pub enum GaugeName { KafkaQueueMessagesBytes, /// The Kafka consumer lag. + #[configurable(metadata(docs::tags = "_component_tags & {topic_id: {description: \"The Kafka topic id.\", required: true}, partition_id: {description: \"The Kafka partition id.\", required: true}}"))] KafkaConsumerLag, /// The total memory currently being used by the Lua runtime. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] LuaMemoryUsedBytes, /// The number of current open connections to Vector. - #[configurable(metadata(docs::tags = "_internal_metrics_tags"))] + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] OpenConnections, /// The number of currently active endpoints. diff --git a/lib/vector-config-macros/src/ast/mod.rs b/lib/vector-config-macros/src/ast/mod.rs index 581671791be5f..f997f3c68a8da 100644 --- a/lib/vector-config-macros/src/ast/mod.rs +++ b/lib/vector-config-macros/src/ast/mod.rs @@ -231,6 +231,12 @@ impl FromMeta for Metadata { }), } } + // Accept path expressions so callers can reference constants: + // `#[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))]` + Expr::Path(path) => Some(LazyCustomAttribute::KeyValue { + key: path_to_string(&nv.path), + value: path.to_token_stream(), + }), expr => { errors .push(darling::Error::unexpected_expr_type(expr).with_span(nmeta)); diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index edf76fdbd9759..df0bc9e6a4ea1 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -19,10 +19,6 @@ components: sources: internal_metrics: { features: { acknowledgements: false - collect: { - checkpoint: enabled: false - from: service: services.vector - } multiline: enabled: false } @@ -57,12 +53,10 @@ components: sources: internal_metrics: { // Metrics emitted by one or more components // Reusable metric definitions - checksum_errors_total: { - tags: _internal_metrics_tags & { - file: _file - } - } connection_read_errors_total: { + description: "The total number of errors reading datagram." + type: "counter" + default_namespace: "vector" tags: _component_tags & { mode: { description: "" @@ -73,328 +67,22 @@ components: sources: internal_metrics: { } } } - events_discarded_total: { - tags: _internal_metrics_tags & { - reason: _reason - } - } - component_discarded_events_total: { - tags: _component_tags & { - intentional: { - description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error." - required: true - } - } - } - component_errors_total: { - tags: _component_tags & { - error_type: _error_type - stage: _stage - } - } - component_received_bytes_total: { - tags: component_received_events_total.tags - } - component_received_bytes: { - tags: component_received_events_total.tags - } - component_received_events_total: { - tags: _component_tags & { - file: { - description: "The file from which the data originated." - required: false - } - uri: { - description: "The sanitized URI from which the data originated." - required: false - } - container_name: { - description: "The name of the container from which the data originated." - required: false - } - pod_name: { - description: "The name of the pod from which the data originated." - required: false - } - peer_addr: { - description: "The IP from which the data originated." - required: false - } - peer_path: { - description: "The pathname from which the data originated." - required: false - } - mode: _mode - } - } - component_received_events_count: { - tags: _component_tags & { - file: { - description: "The file from which the data originated." - required: false - } - uri: { - description: "The sanitized URI from which the data originated." - required: false - } - container_name: { - description: "The name of the container from which the data originated." - required: false - } - pod_name: { - description: "The name of the pod from which the data originated." - required: false - } - peer_addr: { - description: "The IP from which the data originated." - required: false - } - peer_path: { - description: "The pathname from which the data originated." - required: false - } - mode: _mode - } - } - component_received_event_bytes_total: { - tags: component_received_events_total.tags - } - component_sent_bytes_total: { - tags: _component_tags & { - endpoint: { - description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string." - required: false - } - file: { - description: "The absolute path of the destination file." - required: false - } - protocol: { - description: "The protocol used to send the bytes." - required: true - } - region: { - description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname." - required: false - } - } - } - component_sent_events_total: { - tags: _component_tags & {output: _output} - } - component_sent_event_bytes_total: { - tags: _component_tags & {output: _output} - } internal_metrics_cardinality: { + description: "The total number of metrics emitted from the internal metrics registry." + type: "gauge" + default_namespace: "vector" tags: {} } internal_metrics_cardinality_total: { + description: "The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`." + type: "counter" + default_namespace: "vector" tags: internal_metrics_cardinality.tags } - kafka_consumer_lag: { - tags: _component_tags & { - topic_id: { - description: "The Kafka topic id." - required: true - } - partition_id: { - description: "The Kafka partition id." - required: true - } - } - } - files_added_total: { - tags: _internal_metrics_tags & { - file: _file - } - } - files_deleted_total: { - tags: _internal_metrics_tags & { - file: _file - } - } - files_resumed_total: { - tags: _internal_metrics_tags & { - file: _file - } - } - files_unwatched_total: { - tags: _internal_metrics_tags & { - file: _file - } - } - grpc_server_messages_received_total: { - tags: _component_tags & { - grpc_method: _grpc_method - grpc_service: _grpc_service - } - } - grpc_server_messages_sent_total: { - tags: _component_tags & { - grpc_method: _grpc_method - grpc_service: _grpc_service - grpc_status: _grpc_status - } - } - grpc_server_handler_duration_seconds: { - tags: _component_tags & { - grpc_method: _grpc_method - grpc_service: _grpc_service - grpc_status: _grpc_status - } - } - http_client_response_rtt_seconds: { - tags: _component_tags & { - status: _status - } - } - http_client_requests_sent_total: { - tags: _component_tags & { - method: _method - } - } - http_client_responses_total: { - tags: _component_tags & { - status: _status - } - } - http_server_requests_received_total: { - tags: _component_tags & { - method: _method - path: _path - } - } - http_server_responses_sent_total: { - tags: _component_tags & { - method: _method - path: _path - status: _status - } - } - http_server_handler_duration_seconds: { - tags: _component_tags & { - method: _method - path: _path - status: _status - } - } - source_buffer_max_byte_size: { - tags: _component_tags & { - output: _output - } - } - source_buffer_max_event_size: { - tags: _component_tags & { - output: _output - } - } - source_buffer_max_size_bytes: { - tags: _component_tags & { - output: _output - } - } - source_buffer_max_size_events: { - tags: _component_tags & { - output: _output - } - } - source_buffer_utilization: { - tags: _component_tags & { - output: _output - } - } - source_buffer_utilization_level: { - tags: _component_tags & { - output: _output - } - } - source_buffer_utilization_mean: { - tags: _component_tags & { - output: _output - } - } - s3_object_processing_failed_duration_seconds: { - tags: _component_tags & { - bucket: { - description: "The name of the S3 bucket." - required: true - } - } - } - s3_object_processing_succeeded_duration_seconds: { - tags: _component_tags & { - bucket: { - description: "The name of the S3 bucket." - required: true - } - } - } - sqs_s3_event_record_ignored_total: { - - tags: _component_tags & { - ignore_type: { - description: "The reason for ignoring the S3 record" - required: true - enum: { - "invalid_event_kind": "The kind of invalid event." - } - } - } - } - tag_value_limit_exceeded_total: { - tags: _component_tags & { - metric_name: { - description: """ - The name of the metric whose tag value limit was exceeded. - Only present when `internal_metrics.include_extended_tags` is enabled. - """ - required: false - } - tag_key: { - description: """ - The key of the tag whose value limit was exceeded. - Only present when `internal_metrics.include_extended_tags` is enabled. - """ - required: false - } - } - } - transform_buffer_max_byte_size: { - tags: _component_tags & { - output: _output - } - } - transform_buffer_max_event_size: { - tags: _component_tags & { - output: _output - } - } - transform_buffer_max_size_bytes: { - tags: _component_tags & { - output: _output - } - } - transform_buffer_max_size_events: { - tags: _component_tags & { - output: _output - } - } - transform_buffer_utilization: { - tags: _component_tags & { - output: _output - } - } - transform_buffer_utilization_level: { - tags: _component_tags & { - output: _output - } - } - transform_buffer_utilization_mean: { - tags: _component_tags & { - output: _output - } - } utf8_convert_errors_total: { + description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." + type: "counter" + default_namespace: "vector" tags: _component_tags & { mode: { description: "The connection mode used by the component." @@ -405,35 +93,14 @@ components: sources: internal_metrics: { } } } - build_info: { - tags: _internal_metrics_tags & { - debug: { - description: "Whether this is a debug build of Vector" - required: true - } - version: { - description: "Vector version." - required: true - } - rust_version: { - description: "The Rust version from the package manifest." - required: true - } - arch: { - description: "The target architecture being compiled for. (e.g. x86_64)" - required: true - } - revision: { - description: "Revision identifer, related to versioned releases." - required: true - } - } - } // Windows metrics // config metrics config_reload_rejected: { + description: "Number of configuration reload attempts that were rejected." + type: "counter" + default_namespace: "vector" tags: _internal_metrics_tags & { reason: _reason } @@ -448,9 +115,11 @@ components: sources: internal_metrics: { // All available tags _collector: { + description: "Which collector this metric comes from." required: true } _component_kind: { + description: "The Vector component kind." required: true enum: { "sink": "Vector sink components" @@ -459,18 +128,22 @@ components: sources: internal_metrics: { } } _component_id: { + description: "The Vector component ID." required: true examples: ["my_source", "my_sink"] } _component_type: { + description: "The Vector component type." required: true examples: ["file", "http", "honeycomb", "splunk_hec"] } _endpoint: { + description: "The absolute path of originating file." required: true examples: ["http://localhost:8080/server-status?auto"] } _error_type: { + description: "The type of the error" required: true enum: { "acknowledgements_failed": "The acknowledgement operation failed." @@ -499,22 +172,28 @@ components: sources: internal_metrics: { } } _file: { + description: "The file that produced the error" required: false } _grpc_method: { + description: "The name of the method called on the gRPC service." required: true } _grpc_service: { + description: "The gRPC service name." required: true } _grpc_status: { + description: "The human-readable [gRPC status code](\(urls.grpc_status_code))." required: true } _host: { + description: "The hostname of the originating system." required: true examples: [_values.local_host] } _mode: { + description: "The connection mode used by the component." required: false enum: { udp: "User Datagram Protocol" @@ -523,9 +202,11 @@ components: sources: internal_metrics: { } } _output: { + description: "The specific output of the component." required: false } _stage: { + description: "The stage within the component at which the error occurred." required: true enum: { receiving: "While receiving data." @@ -534,15 +215,19 @@ components: sources: internal_metrics: { } } _status: { + description: "The HTTP status code of the request." required: false } _method: { + description: "The HTTP method of the request." required: false } _path: { + description: "The path that produced the error." required: true } _reason: { + description: "The type of the error" required: true enum: { "out_of_order": "The event was out of order." @@ -552,17 +237,5 @@ components: sources: internal_metrics: { } how_it_works: { - unique_series: { - title: "Sending metrics from multiple Vector instances" - body: """ - When sending `internal_metrics` from multiple Vector instances - to the same destination, you will typically want to tag the - metrics with a tag that is unique to the Vector instance sending - the metrics to avoid the metric series conflicting. The - `tags.host_key` option can be used for this, but you can also - use a subsequent `remap` transform to add a different unique - tag from the environment. - """ - } } } From b45fc6ddf9b78a410ecbcafbbb8dac77aeba3094 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 12:02:49 -0400 Subject: [PATCH 16/43] chore(metrics): move remaining non-enum metrics to Rust enums, internal_metrics.cue is now metadata-only --- .../src/internal_event/metric_name.rs | 25 ++++++++ .../components/sources/internal_metrics.cue | 60 +------------------ 2 files changed, 26 insertions(+), 59 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index b94488d72decf..658f940eec356 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -345,6 +345,22 @@ pub enum CounterName { /// The total number of entries evicted from the in-memory enrichment table due to TTL expiration. MemoryEnrichmentTableTtlExpirations, + + /// The total number of errors reading datagram. + #[configurable(metadata(docs::tags = "_component_tags & {mode: {description: \"\", required: true, enum: {udp: \"User Datagram Protocol\"}}}"))] + ConnectionReadErrorsTotal, + + /// The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`. + #[configurable(metadata(docs::tags = "{}"))] + InternalMetricsCardinalityTotal, + + /// Number of configuration reload attempts that were rejected. + #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_REASON))] + ConfigReloadRejected, + + /// The total number of errors converting bytes to a UTF-8 string in UDP mode. + #[configurable(metadata(docs::tags = "_component_tags & {mode: {description: \"The connection mode used by the component.\", required: true, enum: {udp: \"User Datagram Protocol\"}}}"))] + Utf8ConvertErrorsTotal, } /// Canonical list of all per-component internal histogram metric names emitted by Vector. @@ -627,6 +643,10 @@ pub enum GaugeName { /// The number of tag keys currently being tracked by the tag cardinality limit transform. TagCardinalityTrackedKeys, + + /// The total number of metrics emitted from the internal metrics registry. + #[configurable(metadata(docs::tags = "{}"))] + InternalMetricsCardinality, } impl GaugeName { @@ -670,6 +690,7 @@ impl GaugeName { Self::MemoryEnrichmentTableObjectsCount => "memory_enrichment_table_objects_count", Self::MemoryEnrichmentTableByteSize => "memory_enrichment_table_byte_size", Self::TagCardinalityTrackedKeys => "tag_cardinality_tracked_keys", + Self::InternalMetricsCardinality => "internal_metrics_cardinality", } } } @@ -783,6 +804,10 @@ impl CounterName { } Self::MemoryEnrichmentTableReadsTotal => "memory_enrichment_table_reads_total", Self::MemoryEnrichmentTableTtlExpirations => "memory_enrichment_table_ttl_expirations", + Self::ConnectionReadErrorsTotal => "connection_read_errors_total", + Self::InternalMetricsCardinalityTotal => "internal_metrics_cardinality_total", + Self::ConfigReloadRejected => "config_reload_rejected", + Self::Utf8ConvertErrorsTotal => "utf8_convert_errors_total", } } } diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index df0bc9e6a4ea1..38ea524d3bb58 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -49,63 +49,6 @@ components: sources: internal_metrics: { } } - // Instance-level "process" metrics - - // Metrics emitted by one or more components - // Reusable metric definitions - connection_read_errors_total: { - description: "The total number of errors reading datagram." - type: "counter" - default_namespace: "vector" - tags: _component_tags & { - mode: { - description: "" - required: true - enum: { - udp: "User Datagram Protocol" - } - } - } - } - internal_metrics_cardinality: { - description: "The total number of metrics emitted from the internal metrics registry." - type: "gauge" - default_namespace: "vector" - tags: {} - } - internal_metrics_cardinality_total: { - description: "The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`." - type: "counter" - default_namespace: "vector" - tags: internal_metrics_cardinality.tags - } - utf8_convert_errors_total: { - description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." - type: "counter" - default_namespace: "vector" - tags: _component_tags & { - mode: { - description: "The connection mode used by the component." - required: true - enum: { - udp: "User Datagram Protocol" - } - } - } - } - - // Windows metrics - - // config metrics - config_reload_rejected: { - description: "Number of configuration reload attempts that were rejected." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags & { - reason: _reason - } - } - // Helpful tag groupings _component_tags: _internal_metrics_tags & { component_kind: _component_kind @@ -236,6 +179,5 @@ components: sources: internal_metrics: { } } - how_it_works: { - } + how_it_works: {} } From 38fec435d2f571df719395cb9f71a73115d597ce Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 12:40:57 -0400 Subject: [PATCH 17/43] fix(vdev): escape newlines in descriptions before writing CUE string literals --- vdev/src/commands/build/component_docs/runner.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index c3ed78edb3b02..a837ba0294d9c 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -419,7 +419,10 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { entries.push(MetricEntry { name: name.to_owned(), metric_type, - description: desc.replace('\\', "\\\\").replace('"', "\\\""), + description: desc + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n"), tags, deprecated, deprecated_message, @@ -449,7 +452,7 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { if e.deprecated { cue.push_str("\t\tdeprecated: true\n"); if let Some(msg) = &e.deprecated_message { - let escaped = msg.replace('\\', "\\\\").replace('"', "\\\""); + let escaped = msg.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n"); writeln!(cue, "\t\tdeprecated_message: \"{escaped}\"").unwrap(); } } From 40abd88ce0f8a5d1608bf0c8f319915c8e5c1c77 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 12:48:54 -0400 Subject: [PATCH 18/43] Regenerate website/cue/reference/generated/internal_metric_descriptions.cue --- .../internal_metric_descriptions.cue | 1119 +++++++++++++++-- 1 file changed, 983 insertions(+), 136 deletions(-) diff --git a/website/cue/reference/generated/internal_metric_descriptions.cue b/website/cue/reference/generated/internal_metric_descriptions.cue index 5b4b4102bd720..3ad7e27002a6c 100644 --- a/website/cue/reference/generated/internal_metric_descriptions.cue +++ b/website/cue/reference/generated/internal_metric_descriptions.cue @@ -1,140 +1,987 @@ package metadata -// Auto-generated by scripts/generate-internal-metric-docs.sh -// Do not edit manually — update the doc comment on the relevant -// CounterName / HistogramName / GaugeName variant in -// lib/vector-common/src/internal_event/metric_name.rs instead. +// Auto-generated by `vdev build component-docs`. +// Do not edit manually — update the doc comment (or #[configurable(deprecated)] +// attribute) on the relevant CounterName / HistogramName / GaugeName variant in +// lib/vector-common/src/internal_event/metric_name.rs instead, +// then re-run `make generate-component-docs`. components: sources: internal_metrics: output: metrics: { - active_clients: description: "Number of clients attached to a component." - adaptive_concurrency_averaged_rtt: description: "The average round-trip time (RTT) for the current window." - adaptive_concurrency_in_flight: description: "The number of outbound requests currently awaiting a response." - adaptive_concurrency_limit: description: "The concurrency limit that the adaptive concurrency feature has decided on for this current window." - adaptive_concurrency_observed_rtt: description: "The observed round-trip time (RTT) for requests." - aggregate_events_recorded_total: description: "The number of events recorded by the aggregate transform." - aggregate_failed_updates: description: "The number of failed metric updates, `incremental` adds, encountered by the aggregate transform." - aggregate_flushes_total: description: "The number of flushes done by the aggregate transform." - api_started_total: description: "The number of times the Vector API has been started." - buffer_byte_size: description: "The number of bytes currently in the buffer." - buffer_discarded_events_total: description: "The number of events dropped by this non-blocking buffer." - buffer_events: description: "The number of events currently in the buffer." - buffer_received_event_bytes_total: description: "The number of bytes received by this buffer." - buffer_received_events_total: description: "The number of events received by this buffer." - buffer_send_duration_seconds: description: "The duration spent sending a payload to this buffer." - buffer_sent_event_bytes_total: description: "The number of bytes sent by this buffer." - buffer_sent_events_total: description: "The number of events sent by this buffer." - buffer_size_bytes: description: "The number of bytes currently in the buffer." - buffer_size_events: description: "The number of events currently in the buffer." - build_info: description: "Has a fixed value of 1.0. Contains build information such as Rust and Vector versions." - checkpoints_total: description: "The total number of files checkpointed." - checksum_errors_total: description: "The total number of errors identifying files via checksum." - collect_completed_total: description: "The total number of metrics collections completed for this component." - collect_duration_seconds: description: "The duration spent collecting of metrics for this component." - command_executed_total: description: "The total number of times a command has been executed." - command_execution_duration_seconds: description: "The command execution duration in seconds." - component_discarded_events_total: description: "The number of events dropped by this component." - component_errors_total: description: "The total number of errors encountered by this component." - component_latency_mean_seconds: description: "The mean elapsed time, in fractional seconds, that an event spends in a single transform.\n\nThis includes both the time spent queued in the transform’s input buffer and the time spent executing the transform itself.\n\nThis value is smoothed over time using an exponentially weighted moving average (EWMA)." - component_latency_seconds: description: "The elapsed time, in fractional seconds, that an event spends in a single transform.\n\nThis includes both the time spent queued in the transform’s input buffer and the time spent executing the transform itself." - component_received_bytes: description: "The size in bytes of each event received by the source." - component_received_bytes_total: description: "The number of raw bytes accepted by this component from source origins." - component_received_event_bytes_total: description: "The number of event bytes accepted by this component either from\ntagged origins like file and uri, or cumulatively from other origins." - component_received_events_count: description: "A histogram of the number of events passed in each internal batch in Vector's internal topology.\n\nNote that this is separate than sink-level batching. It is mostly useful for low level debugging\nperformance issues in Vector due to small internal batches." - component_received_events_total: description: "The number of events accepted by this component either from tagged\norigins like file and uri, or cumulatively from other origins." - component_sent_bytes_total: description: "The number of raw bytes sent by this component to destination sinks." - component_sent_event_bytes_total: description: "The total number of event bytes emitted by this component." - component_sent_events_total: description: "The total number of events emitted by this component." - component_timed_out_events_total: description: "The total number of events for which this source responded with a timeout error." - component_timed_out_requests_total: description: "The total number of requests for which this source responded with a timeout error." - config_reload_rejected: description: "Number of configuration reload attempts that were rejected." - config_reloaded: description: "Number of times a new configuration was loaded successfully." - connection_established_total: description: "The total number of times a connection has been established." - connection_read_errors_total: description: "The total number of errors reading datagram." - connection_send_errors_total: description: "The total number of errors sending data via the connection." - connection_shutdown_total: description: "The total number of times the connection has been shut down." - container_processed_events_total: description: "The total number of container events processed." - containers_unwatched_total: description: "The total number of times Vector stopped watching for container logs." - containers_watched_total: description: "The total number of times Vector started watching for container logs." - doris_bytes_loaded_total: description: "The total number of bytes loaded into Doris." - doris_rows_filtered_total: description: "The total number of rows filtered by Doris during stream load." - doris_rows_loaded_total: description: "The total number of rows successfully loaded into Doris." - events_discarded_total: description: "The total number of events discarded by this component." - files_added_total: description: "The total number of files Vector has found to watch." - files_deleted_total: description: "The total number of files deleted." - files_resumed_total: description: "The total number of times Vector has resumed watching a file." - files_unwatched_total: description: "The total number of times Vector has stopped watching a file." - grpc_server_handler_duration_seconds: description: "The duration spent handling a gRPC request." - grpc_server_messages_received_total: description: "The total number of gRPC messages received." - grpc_server_messages_sent_total: description: "The total number of gRPC messages sent." - http_client_requests_sent_total: description: "The total number of sent HTTP requests, tagged with the request method." - http_client_response_rtt_seconds: description: "The round-trip time (RTT) of HTTP requests, tagged with the response code." - http_client_responses_total: description: "The total number of HTTP requests, tagged with the response code." - http_client_rtt_seconds: description: "The round-trip time (RTT) of HTTP requests." - http_requests_total: description: "The total number of HTTP requests issued by this component." - http_server_handler_duration_seconds: description: "The duration spent handling a HTTP request." - http_server_requests_received_total: description: "The total number of HTTP requests received." - http_server_responses_sent_total: description: "The total number of HTTP responses sent." - internal_metrics_cardinality: description: "The total number of metrics emitted from the internal metrics registry." - internal_metrics_cardinality_total: description: "The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`." - invalid_record_total: description: "The total number of invalid records that have been discarded." - k8s_docker_format_parse_failures_total: description: "The total number of failures to parse a message as a JSON object." - k8s_format_picker_edge_cases_total: description: "The total number of edge cases encountered while picking format of the Kubernetes log message." - kafka_consumed_messages_bytes_total: description: "Total number of message bytes (including framing) received from Kafka brokers." - kafka_consumed_messages_total: description: "Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers." - kafka_consumer_lag: description: "The Kafka consumer lag." - kafka_produced_messages_bytes_total: description: "Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers." - kafka_produced_messages_total: description: "Total number of messages transmitted (produced) to Kafka brokers." - kafka_queue_messages: description: "Current number of messages in producer queues." - kafka_queue_messages_bytes: description: "Current total size of messages in producer queues." - kafka_requests_bytes_total: description: "Total number of bytes transmitted to Kafka brokers." - kafka_requests_total: description: "Total number of requests sent to Kafka brokers." - kafka_responses_bytes_total: description: "Total number of bytes received from Kafka brokers." - kafka_responses_total: description: "Total number of responses received from Kafka brokers." - lua_memory_used_bytes: description: "The total memory currently being used by the Lua runtime." - metadata_refresh_failed_total: description: "The total number of failed efforts to refresh AWS EC2 metadata." - metadata_refresh_successful_total: description: "The total number of AWS EC2 metadata refreshes." - open_connections: description: "The number of current open connections to Vector." - open_files: description: "The total number of open files." - protobuf_decode_errors_total: description: "The total number of [Protocol Buffers](https://developers.google.com/protocol-buffers) errors thrown during communication between Vector instances." - quit_total: description: "The total number of times the Vector instance has quit." - reloaded_total: description: "The total number of times the Vector instance has been reloaded." - s3_object_processing_failed_duration_seconds: description: "The time taken to process an S3 object that failed, in seconds." - s3_object_processing_succeeded_duration_seconds: description: "The time taken to process an S3 object that succeeded, in seconds." - send_errors_total: description: "The total number of errors sending messages." - source_buffer_max_byte_size: description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." - source_buffer_max_event_size: description: "The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer." - source_buffer_max_size_bytes: description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." - source_buffer_max_size_events: description: "The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer." - source_buffer_utilization: description: "The utilization level of the source buffer. The outputs of the source send data to this buffer." - source_buffer_utilization_level: description: "The current utilization level of the source buffer. The outputs of the source send data to this buffer." - source_buffer_utilization_mean: description: "The mean utilization level of the source buffer. The outputs of the source send data to this buffer. The mean utilization is smoothed over time using an exponentially weighted moving average (EWMA)." - source_lag_time_seconds: description: "The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds." - splunk_pending_acks: description: "The number of outstanding Splunk HEC indexer acknowledgement acks." - sqs_message_delete_succeeded_total: description: "The total number of successful deletions of SQS messages." - sqs_message_processing_succeeded_total: description: "The total number of SQS messages successfully processed." - sqs_message_receive_succeeded_total: description: "The total number of times successfully receiving SQS messages." - sqs_message_received_messages_total: description: "The total number of received SQS messages." - sqs_s3_event_record_ignored_total: description: "The total number of times an S3 record in an SQS message was ignored (for an event that was not `ObjectCreated`)." - stale_events_flushed_total: description: "The number of stale events that Vector has flushed." - started_total: description: "The total number of times the Vector instance has been started." - stdin_reads_failed_total: description: "The total number of errors reading from stdin." - stopped_total: description: "The total number of times the Vector instance has been stopped." - streams_total: description: "The total number of streams." - tag_value_limit_exceeded_total: description: "The total number of events discarded because the tag has been rejected after\nhitting the configured `value_limit`. When `internal_metrics.include_extended_tags`\nis enabled in the `tag_cardinality_limit` transform, this metric includes\n`metric_name` and `tag_key` labels. By default, this metric has no labels to\nkeep cardinality low." - timestamp_parse_errors_total: description: "The total number of errors encountered parsing [RFC 3339](https://tools.ietf.org/html/rfc3339) timestamps." - transform_buffer_max_byte_size: description: "The maximum number of bytes the buffer that feeds into a transform can hold." - transform_buffer_max_event_size: description: "The maximum number of events the buffer that feeds into a transform can hold." - transform_buffer_max_size_bytes: description: "The maximum number of bytes the buffer that feeds into a transform can hold." - transform_buffer_max_size_events: description: "The maximum number of events the buffer that feeds into a transform can hold." - transform_buffer_utilization: description: "The utilization level of the buffer that feeds into a transform." - transform_buffer_utilization_level: description: "The current utilization level of the buffer that feeds into a transform." - transform_buffer_utilization_mean: description: "The mean utilization level of the buffer that feeds into a transform. This value is smoothed over time using an exponentially weighted moving average (EWMA)." - uptime_seconds: description: "The total number of seconds the Vector instance has been up." - utf8_convert_errors_total: description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." - utilization: description: "A ratio from 0 to 1 of the load on a component. A value of 0 would indicate a completely idle component that is simply waiting for input. A value of 1 would indicate a that is never idle. This value is updated every 5 seconds." - value_limit_reached_total: description: "The total number of times new values for a key have been rejected because the\nvalue limit has been reached." - windows_service_install_total: description: "The total number of times the Windows service has been installed." - windows_service_restart_total: description: "The total number of times the Windows service has been restarted." - windows_service_start_total: description: "The total number of times the Windows service has been started." - windows_service_stop_total: description: "The total number of times the Windows service has been stopped." - windows_service_uninstall_total: description: "The total number of times the Windows service has been uninstalled." + active_clients: { + description: "Number of clients attached to a component." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + active_endpoints: { + description: "The number of currently active endpoints." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + adaptive_concurrency_averaged_rtt: { + description: "The average round-trip time (RTT) for the current window." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + adaptive_concurrency_back_pressure: { + description: "The amount of back pressure on the current component." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + adaptive_concurrency_in_flight: { + description: "The number of outbound requests currently awaiting a response." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + adaptive_concurrency_limit: { + description: "The concurrency limit that the adaptive concurrency feature has decided on for this current window." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + adaptive_concurrency_observed_rtt: { + description: "The observed round-trip time (RTT) for requests." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + adaptive_concurrency_past_rtt_mean: { + description: "The mean round-trip time (RTT) for the current window." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + adaptive_concurrency_reached_limit: { + description: "The number of times the concurrency limit was reached." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + aggregate_events_recorded_total: { + description: "The number of events recorded by the aggregate transform." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + aggregate_failed_updates: { + description: "The number of failed metric updates, `incremental` adds, encountered by the aggregate transform." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + aggregate_flushes_total: { + description: "The number of flushes done by the aggregate transform." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + api_started_total: { + description: "The number of times the Vector API has been started." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + buffer_byte_size: { + description: "The number of bytes currently in the buffer." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + deprecated: true + deprecated_message: "This metric has been deprecated in favor of `buffer_size_bytes`." + } + buffer_discarded_bytes_total: { + description: "The number of bytes dropped by this non-blocking buffer." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + buffer_discarded_events_total: { + description: "The number of events dropped by this non-blocking buffer." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + buffer_errors_total: { + description: "The total number of buffer errors encountered." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + buffer_events: { + description: "The number of events currently in the buffer." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + deprecated: true + deprecated_message: "This metric has been deprecated in favor of `buffer_size_events`." + } + buffer_max_byte_size: { + description: "The maximum size in bytes that the buffer can store." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + buffer_max_event_size: { + description: "The maximum size in events that the buffer can store." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + buffer_max_size_bytes: { + description: "The maximum number of bytes in the buffer." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + buffer_max_size_events: { + description: "The maximum number of events in the buffer." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + buffer_received_bytes_total: { + description: "The number of bytes received by this buffer." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + buffer_received_events_total: { + description: "The number of events received by this buffer." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + buffer_send_duration_seconds: { + description: "The duration spent sending a payload to this buffer." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + buffer_sent_bytes_total: { + description: "The number of bytes sent by this buffer." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + buffer_sent_events_total: { + description: "The number of events sent by this buffer." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + buffer_size_bytes: { + description: "The number of bytes currently in the buffer." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + buffer_size_events: { + description: "The number of events currently in the buffer." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + build_info: { + description: "Pseudo-metric that provides build information for the Vector instance." + type: "gauge" + default_namespace: "vector" + tags: _internal_metrics_tags & {debug: {description: "Whether this is a debug build of Vector", required: true}, version: {description: "Vector version.", required: true}, rust_version: {description: "The Rust version from the package manifest.", required: true}, arch: {description: "The target architecture being compiled for. (e.g. x86_64)", required: true}, revision: {description: "Revision identifer, related to versioned releases.", required: true}} + } + checkpoints_total: { + description: "The total number of files checkpointed." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + checksum_errors_total: { + description: "The total number of errors identifying files via checksum." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & {file: _file} + } + collect_completed_total: { + description: "The total number of metrics collections completed for this component." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + collect_duration_seconds: { + description: "The duration spent collecting metrics for this component." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + command_executed_total: { + description: "The total number of times a command has been executed." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + command_execution_duration_seconds: { + description: "The command execution duration in seconds." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + component_allocated_bytes: { + description: "The number of bytes currently allocated by this component." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + component_allocated_bytes_total: { + description: "The total number of bytes allocated by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + component_deallocated_bytes_total: { + description: "The total number of bytes deallocated by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + component_discarded_events_total: { + description: "The number of events dropped by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {intentional: {description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", required: true}} + } + component_errors_total: { + description: "The total number of errors encountered by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {error_type: _error_type, stage: _stage} + } + component_latency_mean_seconds: { + description: "This includes both the time spent queued in the transform's input buffer and the time spent\nexecuting the transform itself. This value is smoothed over time using an exponentially\nweighted moving average (EWMA)." + type: "gauge" + default_namespace: "vector" + tags: _internal_metrics_tags + } + component_latency_seconds: { + description: "This includes both the time spent queued in the transform's input buffer and the time spent\nexecuting the transform itself." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + component_received_bytes: { + description: "The size in bytes of each event received by the source." + type: "histogram" + default_namespace: "vector" + tags: component_received_events_total.tags + } + component_received_bytes_total: { + description: "The number of raw bytes accepted by this component from source origins." + type: "counter" + default_namespace: "vector" + tags: component_received_events_total.tags + } + component_received_event_bytes_total: { + description: "The number of event bytes accepted by this component either from\ntagged origins like file and uri, or cumulatively from other origins." + type: "counter" + default_namespace: "vector" + tags: component_received_events_total.tags + } + component_received_events_count: { + description: "Note that this is separate than sink-level batching. It is mostly useful for low level\ndebugging performance issues in Vector due to small internal batches." + type: "histogram" + default_namespace: "vector" + tags: component_received_events_total.tags + } + component_received_events_total: { + description: "The number of events accepted by this component either from tagged\norigins like file and uri, or cumulatively from other origins." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: _mode} + } + component_sent_bytes_total: { + description: "The number of raw bytes sent by this component to destination sinks." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {endpoint: {description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", required: false}, file: {description: "The absolute path of the destination file.", required: false}, protocol: {description: "The protocol used to send the bytes.", required: true}, region: {description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", required: false}} + } + component_sent_event_bytes_total: { + description: "The total number of event bytes emitted by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + component_sent_events_total: { + description: "The total number of events emitted by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + component_timed_out_events_total: { + description: "The total number of events for which this source responded with a timeout error." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + component_timed_out_requests_total: { + description: "The total number of requests for which this source responded with a timeout error." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + config_reload_rejected: { + description: "Number of configuration reload attempts that were rejected." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & {reason: _reason} + } + connection_established_total: { + description: "The total number of times a connection has been established." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + connection_read_errors_total: { + description: "The total number of errors reading datagram." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {mode: {description: "", required: true, enum: {udp: "User Datagram Protocol"}}} + } + connection_send_errors_total: { + description: "The total number of errors sending data via the connection." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + connection_shutdown_total: { + description: "The total number of times the connection has been shut down." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + container_processed_events_total: { + description: "The total number of container events processed." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + containers_unwatched_total: { + description: "The total number of times Vector stopped watching for container logs." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + containers_watched_total: { + description: "The total number of times Vector started watching for container logs." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + decoder_bom_removals_total: { + description: "The total number of byte order marks (BOM) removed from incoming data." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + decoder_malformed_replacement_warnings_total: { + description: "The total number of warnings when replacing malformed characters during decoding." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + doris_bytes_loaded_total: { + description: "The total number of bytes loaded into Doris." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + doris_rows_filtered_total: { + description: "The total number of rows filtered by Doris during stream load." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + doris_rows_loaded_total: { + description: "The total number of rows successfully loaded into Doris." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + encoder_unmappable_replacement_warnings_total: { + description: "The total number of warnings when replacing unmappable characters during encoding." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + events_discarded_total: { + description: "The total number of events discarded by this component." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & {reason: _reason} + } + files_added_total: { + description: "The total number of files Vector has found to watch." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & {file: _file} + } + files_deleted_total: { + description: "The total number of files deleted." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & {file: _file} + } + files_resumed_total: { + description: "The total number of times Vector has resumed watching a file." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & {file: _file} + } + files_unwatched_total: { + description: "The total number of times Vector has stopped watching a file." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & {file: _file} + } + grpc_server_handler_duration_seconds: { + description: "The duration spent handling a gRPC request." + type: "histogram" + default_namespace: "vector" + tags: _component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service, grpc_status: _grpc_status} + } + grpc_server_messages_received_total: { + description: "The total number of gRPC messages received." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service} + } + grpc_server_messages_sent_total: { + description: "The total number of gRPC messages sent." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service, grpc_status: _grpc_status} + } + http_client_error_rtt_seconds: { + description: "The round-trip time (RTT) of HTTP requests that resulted in an error." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + http_client_errors_total: { + description: "The total number of HTTP client errors encountered." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + http_client_requests_sent_total: { + description: "The total number of sent HTTP requests, tagged with the request method." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {method: _method} + } + http_client_response_rtt_seconds: { + description: "The round-trip time (RTT) of HTTP requests, tagged with the response code." + type: "histogram" + default_namespace: "vector" + tags: _component_tags & {status: _status} + } + http_client_responses_total: { + description: "The total number of HTTP requests, tagged with the response code." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {status: _status} + } + http_client_rtt_seconds: { + description: "The round-trip time (RTT) of HTTP requests." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + http_server_handler_duration_seconds: { + description: "The duration spent handling an HTTP request." + type: "histogram" + default_namespace: "vector" + tags: _component_tags & {method: _method, path: _path, status: _status} + } + http_server_requests_received_total: { + description: "The total number of HTTP requests received." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {method: _method, path: _path} + } + http_server_responses_sent_total: { + description: "The total number of HTTP responses sent." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {method: _method, path: _path, status: _status} + } + internal_metrics_cardinality: { + description: "The total number of metrics emitted from the internal metrics registry." + type: "gauge" + default_namespace: "vector" + tags: {} + } + internal_metrics_cardinality_total: { + description: "The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`." + type: "counter" + default_namespace: "vector" + tags: {} + } + k8s_docker_format_parse_failures_total: { + description: "The total number of failures to parse a message as a JSON object." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + k8s_event_namespace_annotation_failures_total: { + description: "The total number of failures to annotate Kubernetes events with namespace metadata." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + k8s_event_node_annotation_failures_total: { + description: "The total number of failures to annotate Kubernetes events with node metadata." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + k8s_format_picker_edge_cases_total: { + description: "The total number of edge cases encountered while picking format of the Kubernetes log message." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + kafka_consumed_messages_bytes_total: { + description: "Total number of message bytes (including framing) received from Kafka brokers." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + kafka_consumed_messages_total: { + description: "Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + kafka_consumer_lag: { + description: "The Kafka consumer lag." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {topic_id: {description: "The Kafka topic id.", required: true}, partition_id: {description: "The Kafka partition id.", required: true}} + } + kafka_produced_messages_bytes_total: { + description: "Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + kafka_produced_messages_total: { + description: "Total number of messages transmitted (produced) to Kafka brokers." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + kafka_queue_messages: { + description: "Current number of messages in producer queues." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + kafka_queue_messages_bytes: { + description: "Current total size of messages in producer queues." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + kafka_requests_bytes_total: { + description: "Total number of bytes transmitted to Kafka brokers." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + kafka_requests_total: { + description: "Total number of requests sent to Kafka brokers." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + kafka_responses_bytes_total: { + description: "Total number of bytes received from Kafka brokers." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + kafka_responses_total: { + description: "Total number of responses received from Kafka brokers." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + lua_memory_used_bytes: { + description: "The total memory currently being used by the Lua runtime." + type: "gauge" + default_namespace: "vector" + tags: _internal_metrics_tags + } + memory_enrichment_table_byte_size: { + description: "The total size in bytes of all objects stored in the in-memory enrichment table." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + memory_enrichment_table_failed_insertions: { + description: "The total number of failed insertions into the in-memory enrichment table." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + memory_enrichment_table_failed_reads: { + description: "The total number of failed reads from the in-memory enrichment table." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + memory_enrichment_table_flushes_total: { + description: "The total number of flushes of the in-memory enrichment table." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + memory_enrichment_table_insertions_total: { + description: "The total number of successful insertions into the in-memory enrichment table." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + memory_enrichment_table_objects_count: { + description: "The number of objects currently stored in the in-memory enrichment table." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + memory_enrichment_table_reads_total: { + description: "The total number of successful reads from the in-memory enrichment table." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + memory_enrichment_table_ttl_expirations: { + description: "The total number of entries evicted from the in-memory enrichment table due to TTL expiration." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + metadata_refresh_failed_total: { + description: "The total number of failed efforts to refresh AWS EC2 metadata." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + metadata_refresh_successful_total: { + description: "The total number of AWS EC2 metadata refreshes." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + open_connections: { + description: "The number of current open connections to Vector." + type: "gauge" + default_namespace: "vector" + tags: _internal_metrics_tags + } + open_files: { + description: "The total number of open files." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + parse_errors_total: { + description: "The total number of errors encountered while parsing." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + quit_total: { + description: "The total number of times the Vector instance has quit." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + reloaded_total: { + description: "The total number of times the Vector instance has been reloaded." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + rewritten_timestamp_events_total: { + description: "The total number of events with rewrapped timestamps." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + s3_object_processing_failed_duration_seconds: { + description: "The time taken to process an S3 object that failed, in seconds." + type: "histogram" + default_namespace: "vector" + tags: _component_tags & {bucket: {description: "The name of the S3 bucket.", required: true}} + } + s3_object_processing_succeeded_duration_seconds: { + description: "The time taken to process an S3 object that succeeded, in seconds." + type: "histogram" + default_namespace: "vector" + tags: _component_tags & {bucket: {description: "The name of the S3 bucket.", required: true}} + } + source_buffer_max_byte_size: { + description: "The maximum number of bytes the source buffer can hold." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + deprecated: true + deprecated_message: "This metric has been deprecated in favor of `source_buffer_max_size_bytes`." + } + source_buffer_max_event_size: { + description: "The maximum number of events the source buffer can hold." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + deprecated: true + deprecated_message: "This metric has been deprecated in favor of `source_buffer_max_size_events`." + } + source_buffer_max_size_bytes: { + description: "The maximum number of bytes the source buffer can hold." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + source_buffer_max_size_events: { + description: "The maximum number of events the source buffer can hold." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + source_buffer_utilization: { + description: "The utilization of the source buffer." + type: "histogram" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + source_buffer_utilization_level: { + description: "The current utilization level of the source buffer." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + source_buffer_utilization_mean: { + description: "The mean utilization of the source buffer, smoothed with an EWMA." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + source_lag_time_seconds: { + description: "The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + source_send_batch_latency_seconds: { + description: "The time elapsed blocking on the downstream channel to accept an entire batch of events received at the source." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + source_send_latency_seconds: { + description: "The time elapsed blocking on the downstream channel to accept a single chunk from a batch of events received at the source." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + splunk_pending_acks: { + description: "The number of outstanding Splunk HEC indexer acknowledgement acks." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_defer_succeeded_total: { + description: "The total number of successful deferrals of SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_delete_succeeded_total: { + description: "The total number of successful deletions of SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_processing_succeeded_total: { + description: "The total number of SQS messages successfully processed." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_receive_succeeded_total: { + description: "The total number of times successfully receiving SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_received_messages_total: { + description: "The total number of received SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_s3_event_record_ignored_total: { + description: "The total number of times an S3 record in an SQS message was ignored." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {ignore_type: {description: "The reason for ignoring the S3 record", required: true, enum: {invalid_event_kind: "The kind of invalid event."}}} + } + stale_events_flushed_total: { + description: "The number of stale events that Vector has flushed." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + started_total: { + description: "The total number of times the Vector instance has been started." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + stopped_total: { + description: "The total number of times the Vector instance has been stopped." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + tag_cardinality_tracked_keys: { + description: "The number of tag keys currently being tracked by the tag cardinality limit transform." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + tag_cardinality_untracked_events_total: { + description: "The total number of events that contained a tag which exceeded the configured cardinality limit." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + tag_value_limit_exceeded_total: { + description: "The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {metric_name: {description: "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", required: false}, tag_key: {description: "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", required: false}} + } + transform_buffer_max_byte_size: { + description: "The maximum number of bytes the buffer that feeds into a transform can hold." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + deprecated: true + deprecated_message: "This metric has been deprecated in favor of `transform_buffer_max_size_bytes`." + } + transform_buffer_max_event_size: { + description: "The maximum number of events the buffer that feeds into a transform can hold." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + deprecated: true + deprecated_message: "This metric has been deprecated in favor of `transform_buffer_max_size_events`." + } + transform_buffer_max_size_bytes: { + description: "The maximum number of bytes the buffer that feeds into a transform can hold." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + transform_buffer_max_size_events: { + description: "The maximum number of events the buffer that feeds into a transform can hold." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + transform_buffer_utilization: { + description: "The utilization of the buffer that feeds into a transform." + type: "histogram" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + transform_buffer_utilization_level: { + description: "The current utilization level of the buffer that feeds into a transform." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + transform_buffer_utilization_mean: { + description: "The mean utilization of the buffer that feeds into a transform, smoothed with an EWMA." + type: "gauge" + default_namespace: "vector" + tags: _component_tags & {output: _output} + } + uptime_seconds: { + description: "The number of seconds the Vector instance has been running." + type: "gauge" + default_namespace: "vector" + tags: _internal_metrics_tags + } + utf8_convert_errors_total: { + description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." + type: "counter" + default_namespace: "vector" + tags: _component_tags & {mode: {description: "The connection mode used by the component.", required: true, enum: {udp: "User Datagram Protocol"}}} + } + utilization: { + description: "The current utilization of this component, expressed as a value from 0 to 1." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + value_limit_reached_total: { + description: "The total number of times the value limit was reached." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + websocket_bytes_sent_total: { + description: "The total number of bytes sent over WebSocket connections." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + websocket_messages_sent_total: { + description: "The total number of messages sent over WebSocket connections." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + windows_service_install_total: { + description: "The total number of times the Windows service has been installed." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + windows_service_restart_total: { + description: "The total number of times the Windows service has been restarted." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + windows_service_start_total: { + description: "The total number of times the Windows service has been started." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + windows_service_stop_total: { + description: "The total number of times the Windows service has been stopped." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + windows_service_uninstall_total: { + description: "The total number of times the Windows service has been uninstalled." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } } From d47eeb684c41598697673cf8874c4585a035a859 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 12:53:44 -0400 Subject: [PATCH 19/43] chore(metrics): replace raw CUE tag strings with typed TagSet struct and compile-time constants --- .../src/internal_event/metric_name.rs | 54 +-- .../src/internal_event/metric_tags.rs | 431 ++++++++++++++++++ lib/vector-common/src/internal_event/mod.rs | 1 + 3 files changed, 456 insertions(+), 30 deletions(-) create mode 100644 lib/vector-common/src/internal_event/metric_tags.rs diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index 658f940eec356..8aa8af737a737 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -1,22 +1,16 @@ use strum::{AsRefStr, Display, EnumIter}; use vector_config::configurable_component; -// CUE tag-set constants referenced via `#[configurable(metadata(docs::tags = CONSTANT))]`. -// The values are raw CUE expressions emitted verbatim by `vdev build component-docs`. -pub const COMPONENT_TAGS: &str = "_component_tags"; -pub const INTERNAL_METRICS_TAGS: &str = "_internal_metrics_tags"; -pub const COMPONENT_TAGS_OUTPUT: &str = "_component_tags & {output: _output}"; -pub const INTERNAL_METRICS_TAGS_FILE: &str = "_internal_metrics_tags & {file: _file}"; -pub const INTERNAL_METRICS_TAGS_REASON: &str = "_internal_metrics_tags & {reason: _reason}"; -pub const COMPONENT_TAGS_GRPC_METHOD_SERVICE: &str = - "_component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service}"; -pub const COMPONENT_TAGS_GRPC_ALL: &str = "_component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service, grpc_status: _grpc_status}"; -pub const COMPONENT_TAGS_HTTP_METHOD: &str = "_component_tags & {method: _method}"; -pub const COMPONENT_TAGS_HTTP_STATUS: &str = "_component_tags & {status: _status}"; -pub const COMPONENT_TAGS_HTTP_METHOD_PATH: &str = "_component_tags & {method: _method, path: _path}"; -pub const COMPONENT_TAGS_HTTP_ALL: &str = - "_component_tags & {method: _method, path: _path, status: _status}"; -pub const COMPONENT_RECEIVED_EVENTS_TAGS: &str = "component_received_events_total.tags"; +use super::metric_tags::{ + BUILD_INFO_TAGS, COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS, COMPONENT_RECEIVED_EVENTS_TAGS, + COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS, COMPONENT_SENT_BYTES_TOTAL_TAGS, + COMPONENT_TAGS_ERROR_TYPE_STAGE, COMPONENT_TAGS_GRPC_ALL, COMPONENT_TAGS_GRPC_METHOD_SERVICE, + COMPONENT_TAGS_HTTP_ALL, COMPONENT_TAGS_HTTP_METHOD, COMPONENT_TAGS_HTTP_METHOD_PATH, + COMPONENT_TAGS_HTTP_STATUS, COMPONENT_TAGS_OUTPUT, CONNECTION_READ_ERRORS_TOTAL_TAGS, + EMPTY_TAGS, INTERNAL_METRICS_TAGS, INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, + KAFKA_CONSUMER_LAG_TAGS, S3_OBJECT_PROCESSING_TAGS, SQS_S3_IGNORED_TOTAL_TAGS, + TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS, UTF8_CONVERT_ERRORS_TOTAL_TAGS, +}; /// Canonical list of all per-component internal counter metric names emitted by Vector. #[configurable_component] @@ -26,7 +20,7 @@ pub const COMPONENT_RECEIVED_EVENTS_TAGS: &str = "component_received_events_tota pub enum CounterName { /// The number of events accepted by this component either from tagged /// origins like file and uri, or cumulatively from other origins. - #[configurable(metadata(docs::tags = "_component_tags & {file: {description: \"The file from which the data originated.\", required: false}, uri: {description: \"The sanitized URI from which the data originated.\", required: false}, container_name: {description: \"The name of the container from which the data originated.\", required: false}, pod_name: {description: \"The name of the pod from which the data originated.\", required: false}, peer_addr: {description: \"The IP from which the data originated.\", required: false}, peer_path: {description: \"The pathname from which the data originated.\", required: false}, mode: _mode}"))] + #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS))] ComponentReceivedEventsTotal, /// The number of event bytes accepted by this component either from @@ -47,15 +41,15 @@ pub enum CounterName { ComponentSentEventBytesTotal, /// The number of raw bytes sent by this component to destination sinks. - #[configurable(metadata(docs::tags = "_component_tags & {endpoint: {description: \"The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.\", required: false}, file: {description: \"The absolute path of the destination file.\", required: false}, protocol: {description: \"The protocol used to send the bytes.\", required: true}, region: {description: \"The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.\", required: false}}"))] + #[configurable(metadata(docs::tags = COMPONENT_SENT_BYTES_TOTAL_TAGS))] ComponentSentBytesTotal, /// The number of events dropped by this component. - #[configurable(metadata(docs::tags = "_component_tags & {intentional: {description: \"True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.\", required: true}}"))] + #[configurable(metadata(docs::tags = COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS))] ComponentDiscardedEventsTotal, /// The total number of errors encountered by this component. - #[configurable(metadata(docs::tags = "_component_tags & {error_type: _error_type, stage: _stage}"))] + #[configurable(metadata(docs::tags = COMPONENT_TAGS_ERROR_TYPE_STAGE))] ComponentErrorsTotal, /// The total number of events for which this source responded with a timeout error. @@ -274,7 +268,7 @@ pub enum CounterName { TagCardinalityUntrackedEventsTotal, /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. - #[configurable(metadata(docs::tags = "_component_tags & {metric_name: {description: \"The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.\", required: false}, tag_key: {description: \"The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.\", required: false}}"))] + #[configurable(metadata(docs::tags = TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS))] TagValueLimitExceededTotal, /// The total number of times the value limit was reached. @@ -319,7 +313,7 @@ pub enum CounterName { K8sDockerFormatParseFailuresTotal, /// The total number of times an S3 record in an SQS message was ignored. - #[configurable(metadata(docs::tags = "_component_tags & {ignore_type: {description: \"The reason for ignoring the S3 record\", required: true, enum: {invalid_event_kind: \"The kind of invalid event.\"}}}"))] + #[configurable(metadata(docs::tags = SQS_S3_IGNORED_TOTAL_TAGS))] SqsS3EventRecordIgnoredTotal, /// The total number of bytes allocated by this component. @@ -347,11 +341,11 @@ pub enum CounterName { MemoryEnrichmentTableTtlExpirations, /// The total number of errors reading datagram. - #[configurable(metadata(docs::tags = "_component_tags & {mode: {description: \"\", required: true, enum: {udp: \"User Datagram Protocol\"}}}"))] + #[configurable(metadata(docs::tags = CONNECTION_READ_ERRORS_TOTAL_TAGS))] ConnectionReadErrorsTotal, /// The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`. - #[configurable(metadata(docs::tags = "{}"))] + #[configurable(metadata(docs::tags = EMPTY_TAGS))] InternalMetricsCardinalityTotal, /// Number of configuration reload attempts that were rejected. @@ -359,7 +353,7 @@ pub enum CounterName { ConfigReloadRejected, /// The total number of errors converting bytes to a UTF-8 string in UDP mode. - #[configurable(metadata(docs::tags = "_component_tags & {mode: {description: \"The connection mode used by the component.\", required: true, enum: {udp: \"User Datagram Protocol\"}}}"))] + #[configurable(metadata(docs::tags = UTF8_CONVERT_ERRORS_TOTAL_TAGS))] Utf8ConvertErrorsTotal, } @@ -421,11 +415,11 @@ pub enum HistogramName { AdaptiveConcurrencyReachedLimit, /// The time taken to process an S3 object that succeeded, in seconds. - #[configurable(metadata(docs::tags = "_component_tags & {bucket: {description: \"The name of the S3 bucket.\", required: true}}"))] + #[configurable(metadata(docs::tags = S3_OBJECT_PROCESSING_TAGS))] S3ObjectProcessingSucceededDurationSeconds, /// The time taken to process an S3 object that failed, in seconds. - #[configurable(metadata(docs::tags = "_component_tags & {bucket: {description: \"The name of the S3 bucket.\", required: true}}"))] + #[configurable(metadata(docs::tags = S3_OBJECT_PROCESSING_TAGS))] S3ObjectProcessingFailedDurationSeconds, /// The duration spent collecting metrics for this component. @@ -605,7 +599,7 @@ pub enum GaugeName { UptimeSeconds, /// Pseudo-metric that provides build information for the Vector instance. - #[configurable(metadata(docs::tags = "_internal_metrics_tags & {debug: {description: \"Whether this is a debug build of Vector\", required: true}, version: {description: \"Vector version.\", required: true}, rust_version: {description: \"The Rust version from the package manifest.\", required: true}, arch: {description: \"The target architecture being compiled for. (e.g. x86_64)\", required: true}, revision: {description: \"Revision identifer, related to versioned releases.\", required: true}}"))] + #[configurable(metadata(docs::tags = BUILD_INFO_TAGS))] BuildInfo, /// Current number of messages in producer queues. @@ -615,7 +609,7 @@ pub enum GaugeName { KafkaQueueMessagesBytes, /// The Kafka consumer lag. - #[configurable(metadata(docs::tags = "_component_tags & {topic_id: {description: \"The Kafka topic id.\", required: true}, partition_id: {description: \"The Kafka partition id.\", required: true}}"))] + #[configurable(metadata(docs::tags = KAFKA_CONSUMER_LAG_TAGS))] KafkaConsumerLag, /// The total memory currently being used by the Lua runtime. @@ -645,7 +639,7 @@ pub enum GaugeName { TagCardinalityTrackedKeys, /// The total number of metrics emitted from the internal metrics registry. - #[configurable(metadata(docs::tags = "{}"))] + #[configurable(metadata(docs::tags = EMPTY_TAGS))] InternalMetricsCardinality, } diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs new file mode 100644 index 0000000000000..9f872ae866d73 --- /dev/null +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -0,0 +1,431 @@ +use serde::Serialize; + +/// A complete tag set for a Vector internal metric, represented as structured +/// data that is serialized to a CUE expression at schema-generation time. +/// +/// Use the pre-defined constants (`COMPONENT_TAGS`, `COMPONENT_TAGS_OUTPUT`, +/// etc.) or construct one inline for metric-specific tag shapes. +#[derive(Clone, Copy)] +pub struct TagSet { + /// Base CUE tag group (e.g. `"_component_tags"`). `None` means an empty + /// base (`{}`). + pub base: Option<&'static str>, + /// Extra fields merged into the base with `&`. + pub extra: &'static [ExtraTag], +} + +/// A single extra tag field appended to a [`TagSet`]. +#[derive(Clone, Copy)] +pub struct ExtraTag { + pub name: &'static str, + pub field: TagField, +} + +/// Definition of one tag field. +#[derive(Clone, Copy)] +pub enum TagField { + /// Reference to an existing CUE tag helper variable (e.g. `_output`). + Ref(&'static str), + /// Inline field definition with an optional fixed-value enum. + Inline { + description: &'static str, + required: bool, + enum_values: Option<&'static [(&'static str, &'static str)]>, + }, +} + +impl TagSet { + fn format_extras(self) -> Option { + if self.extra.is_empty() { + return None; + } + let parts: Vec = self + .extra + .iter() + .map(|et| { + let field = match et.field { + TagField::Ref(r) => r.to_owned(), + TagField::Inline { + description, + required, + enum_values: None, + } => format!(r#"{{description: "{description}", required: {required}}}"#), + TagField::Inline { + description, + required, + enum_values: Some(enums), + } => { + let pairs: Vec = + enums.iter().map(|(k, v)| format!("{k}: \"{v}\"")).collect(); + format!( + r#"{{description: "{description}", required: {required}, enum: {{{}}}}}"#, + pairs.join(", ") + ) + } + }; + format!("{}: {field}", et.name) + }) + .collect(); + Some(parts.join(", ")) + } + + /// Renders the tag set as a CUE expression (emitted verbatim by vdev). + #[must_use] + pub fn to_cue(self) -> String { + match (self.base, self.format_extras().as_deref()) { + (None, None) => "{}".to_owned(), + (Some(base), None) => base.to_owned(), + (None, Some(ex)) => format!("{{{ex}}}"), + (Some(base), Some(ex)) => format!("{base} & {{{ex}}}"), + } + } +} + +impl Serialize for TagSet { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&self.to_cue()) + } +} + +// ─── Common base tag sets ────────────────────────────────────────────────────── + +pub const COMPONENT_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[], +}; + +pub const INTERNAL_METRICS_TAGS: TagSet = TagSet { + base: Some("_internal_metrics_tags"), + extra: &[], +}; + +pub const EMPTY_TAGS: TagSet = TagSet { + base: None, + extra: &[], +}; + +// ─── Cross-reference ────────────────────────────────────────────────────────── + +/// Inherits the tag set defined on `component_received_events_total`. +pub const COMPONENT_RECEIVED_EVENTS_TAGS: TagSet = TagSet { + base: Some("component_received_events_total.tags"), + extra: &[], +}; + +// ─── Component tags with a single CUE-helper extra field ────────────────────── + +pub const COMPONENT_TAGS_OUTPUT: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ExtraTag { name: "output", field: TagField::Ref("_output") }], +}; + +pub const INTERNAL_METRICS_TAGS_FILE: TagSet = TagSet { + base: Some("_internal_metrics_tags"), + extra: &[ExtraTag { name: "file", field: TagField::Ref("_file") }], +}; + +pub const INTERNAL_METRICS_TAGS_REASON: TagSet = TagSet { + base: Some("_internal_metrics_tags"), + extra: &[ExtraTag { name: "reason", field: TagField::Ref("_reason") }], +}; + +pub const COMPONENT_TAGS_ERROR_TYPE_STAGE: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ + ExtraTag { name: "error_type", field: TagField::Ref("_error_type") }, + ExtraTag { name: "stage", field: TagField::Ref("_stage") }, + ], +}; + +// ─── gRPC tag sets ──────────────────────────────────────────────────────────── + +pub const COMPONENT_TAGS_GRPC_METHOD_SERVICE: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ + ExtraTag { name: "grpc_method", field: TagField::Ref("_grpc_method") }, + ExtraTag { name: "grpc_service", field: TagField::Ref("_grpc_service") }, + ], +}; + +pub const COMPONENT_TAGS_GRPC_ALL: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ + ExtraTag { name: "grpc_method", field: TagField::Ref("_grpc_method") }, + ExtraTag { name: "grpc_service", field: TagField::Ref("_grpc_service") }, + ExtraTag { name: "grpc_status", field: TagField::Ref("_grpc_status") }, + ], +}; + +// ─── HTTP tag sets ──────────────────────────────────────────────────────────── + +pub const COMPONENT_TAGS_HTTP_METHOD: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ExtraTag { name: "method", field: TagField::Ref("_method") }], +}; + +pub const COMPONENT_TAGS_HTTP_STATUS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ExtraTag { name: "status", field: TagField::Ref("_status") }], +}; + +pub const COMPONENT_TAGS_HTTP_METHOD_PATH: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ + ExtraTag { name: "method", field: TagField::Ref("_method") }, + ExtraTag { name: "path", field: TagField::Ref("_path") }, + ], +}; + +pub const COMPONENT_TAGS_HTTP_ALL: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ + ExtraTag { name: "method", field: TagField::Ref("_method") }, + ExtraTag { name: "path", field: TagField::Ref("_path") }, + ExtraTag { name: "status", field: TagField::Ref("_status") }, + ], +}; + +// ─── Metric-specific tag sets ───────────────────────────────────────────────── + +pub const COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ + ExtraTag { + name: "file", + field: TagField::Inline { + description: "The file from which the data originated.", + required: false, + enum_values: None, + }, + }, + ExtraTag { + name: "uri", + field: TagField::Inline { + description: "The sanitized URI from which the data originated.", + required: false, + enum_values: None, + }, + }, + ExtraTag { + name: "container_name", + field: TagField::Inline { + description: "The name of the container from which the data originated.", + required: false, + enum_values: None, + }, + }, + ExtraTag { + name: "pod_name", + field: TagField::Inline { + description: "The name of the pod from which the data originated.", + required: false, + enum_values: None, + }, + }, + ExtraTag { + name: "peer_addr", + field: TagField::Inline { + description: "The IP from which the data originated.", + required: false, + enum_values: None, + }, + }, + ExtraTag { + name: "peer_path", + field: TagField::Inline { + description: "The pathname from which the data originated.", + required: false, + enum_values: None, + }, + }, + ExtraTag { name: "mode", field: TagField::Ref("_mode") }, + ], +}; + +pub const COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ExtraTag { + name: "intentional", + field: TagField::Inline { + description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", + required: true, + enum_values: None, + }, + }], +}; + +pub const COMPONENT_SENT_BYTES_TOTAL_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ + ExtraTag { + name: "endpoint", + field: TagField::Inline { + description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", + required: false, + enum_values: None, + }, + }, + ExtraTag { + name: "file", + field: TagField::Inline { + description: "The absolute path of the destination file.", + required: false, + enum_values: None, + }, + }, + ExtraTag { + name: "protocol", + field: TagField::Inline { + description: "The protocol used to send the bytes.", + required: true, + enum_values: None, + }, + }, + ExtraTag { + name: "region", + field: TagField::Inline { + description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", + required: false, + enum_values: None, + }, + }, + ], +}; + +pub const S3_OBJECT_PROCESSING_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ExtraTag { + name: "bucket", + field: TagField::Inline { + description: "The name of the S3 bucket.", + required: true, + enum_values: None, + }, + }], +}; + +pub const KAFKA_CONSUMER_LAG_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ + ExtraTag { + name: "topic_id", + field: TagField::Inline { + description: "The Kafka topic id.", + required: true, + enum_values: None, + }, + }, + ExtraTag { + name: "partition_id", + field: TagField::Inline { + description: "The Kafka partition id.", + required: true, + enum_values: None, + }, + }, + ], +}; + +pub const TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ + ExtraTag { + name: "metric_name", + field: TagField::Inline { + description: "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", + required: false, + enum_values: None, + }, + }, + ExtraTag { + name: "tag_key", + field: TagField::Inline { + description: "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", + required: false, + enum_values: None, + }, + }, + ], +}; + +pub const SQS_S3_IGNORED_TOTAL_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ExtraTag { + name: "ignore_type", + field: TagField::Inline { + description: "The reason for ignoring the S3 record", + required: true, + enum_values: Some(&[("invalid_event_kind", "The kind of invalid event.")]), + }, + }], +}; + +pub const BUILD_INFO_TAGS: TagSet = TagSet { + base: Some("_internal_metrics_tags"), + extra: &[ + ExtraTag { + name: "debug", + field: TagField::Inline { + description: "Whether this is a debug build of Vector", + required: true, + enum_values: None, + }, + }, + ExtraTag { + name: "version", + field: TagField::Inline { + description: "Vector version.", + required: true, + enum_values: None, + }, + }, + ExtraTag { + name: "rust_version", + field: TagField::Inline { + description: "The Rust version from the package manifest.", + required: true, + enum_values: None, + }, + }, + ExtraTag { + name: "arch", + field: TagField::Inline { + description: "The target architecture being compiled for. (e.g. x86_64)", + required: true, + enum_values: None, + }, + }, + ExtraTag { + name: "revision", + field: TagField::Inline { + description: "Revision identifer, related to versioned releases.", + required: true, + enum_values: None, + }, + }, + ], +}; + +pub const CONNECTION_READ_ERRORS_TOTAL_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ExtraTag { + name: "mode", + field: TagField::Inline { + description: "", + required: true, + enum_values: Some(&[("udp", "User Datagram Protocol")]), + }, + }], +}; + +pub const UTF8_CONVERT_ERRORS_TOTAL_TAGS: TagSet = TagSet { + base: Some("_component_tags"), + extra: &[ExtraTag { + name: "mode", + field: TagField::Inline { + description: "The connection mode used by the component.", + required: true, + enum_values: Some(&[("udp", "User Datagram Protocol")]), + }, + }], +}; diff --git a/lib/vector-common/src/internal_event/mod.rs b/lib/vector-common/src/internal_event/mod.rs index e776e89553172..822299ccf19e2 100644 --- a/lib/vector-common/src/internal_event/mod.rs +++ b/lib/vector-common/src/internal_event/mod.rs @@ -6,6 +6,7 @@ pub mod component_events_timed_out; mod events_received; mod events_sent; pub mod metric_name; +pub mod metric_tags; mod optional_tag; mod prelude; pub mod service; From 35c5871c2610b55e04d4f575cd98124a5014abb9 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 12:59:01 -0400 Subject: [PATCH 20/43] chore(metrics): replace TagSet abstraction with serde_json::json!() for inline tag shapes --- .../src/internal_event/metric_name.rs | 59 ++- .../src/internal_event/metric_tags.rs | 431 ------------------ lib/vector-common/src/internal_event/mod.rs | 1 - lib/vector-config-macros/src/ast/mod.rs | 5 + .../commands/build/component_docs/runner.rs | 79 +++- 5 files changed, 114 insertions(+), 461 deletions(-) delete mode 100644 lib/vector-common/src/internal_event/metric_tags.rs diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index 8aa8af737a737..a007c5c5c1973 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -1,16 +1,29 @@ +use serde_json::json; use strum::{AsRefStr, Display, EnumIter}; use vector_config::configurable_component; -use super::metric_tags::{ - BUILD_INFO_TAGS, COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS, COMPONENT_RECEIVED_EVENTS_TAGS, - COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS, COMPONENT_SENT_BYTES_TOTAL_TAGS, - COMPONENT_TAGS_ERROR_TYPE_STAGE, COMPONENT_TAGS_GRPC_ALL, COMPONENT_TAGS_GRPC_METHOD_SERVICE, - COMPONENT_TAGS_HTTP_ALL, COMPONENT_TAGS_HTTP_METHOD, COMPONENT_TAGS_HTTP_METHOD_PATH, - COMPONENT_TAGS_HTTP_STATUS, COMPONENT_TAGS_OUTPUT, CONNECTION_READ_ERRORS_TOTAL_TAGS, - EMPTY_TAGS, INTERNAL_METRICS_TAGS, INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, - KAFKA_CONSUMER_LAG_TAGS, S3_OBJECT_PROCESSING_TAGS, SQS_S3_IGNORED_TOTAL_TAGS, - TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS, UTF8_CONVERT_ERRORS_TOTAL_TAGS, -}; +// CUE tag-set constants for patterns shared across multiple metrics. +// Values are verbatim CUE expressions emitted by `vdev build component-docs`. +// Inline `json!()` is used directly on variants whose tag shape is unique. +pub const COMPONENT_TAGS: &str = "_component_tags"; +pub const INTERNAL_METRICS_TAGS: &str = "_internal_metrics_tags"; +pub const COMPONENT_TAGS_OUTPUT: &str = "_component_tags & {output: _output}"; +pub const INTERNAL_METRICS_TAGS_FILE: &str = "_internal_metrics_tags & {file: _file}"; +pub const INTERNAL_METRICS_TAGS_REASON: &str = "_internal_metrics_tags & {reason: _reason}"; +pub const COMPONENT_TAGS_GRPC_METHOD_SERVICE: &str = + "_component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service}"; +pub const COMPONENT_TAGS_GRPC_ALL: &str = + "_component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service, grpc_status: _grpc_status}"; +pub const COMPONENT_TAGS_HTTP_METHOD: &str = "_component_tags & {method: _method}"; +pub const COMPONENT_TAGS_HTTP_STATUS: &str = "_component_tags & {status: _status}"; +pub const COMPONENT_TAGS_HTTP_METHOD_PATH: &str = + "_component_tags & {method: _method, path: _path}"; +pub const COMPONENT_TAGS_HTTP_ALL: &str = + "_component_tags & {method: _method, path: _path, status: _status}"; +pub const COMPONENT_TAGS_ERROR_TYPE_STAGE: &str = + "_component_tags & {error_type: _error_type, stage: _stage}"; +/// Inherits the tag set defined on `component_received_events_total`. +pub const COMPONENT_RECEIVED_EVENTS_TAGS: &str = "component_received_events_total.tags"; /// Canonical list of all per-component internal counter metric names emitted by Vector. #[configurable_component] @@ -20,7 +33,7 @@ use super::metric_tags::{ pub enum CounterName { /// The number of events accepted by this component either from tagged /// origins like file and uri, or cumulatively from other origins. - #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"file": {"description": "The file from which the data originated.", "required": false}, "uri": {"description": "The sanitized URI from which the data originated.", "required": false}, "container_name": {"description": "The name of the container from which the data originated.", "required": false}, "pod_name": {"description": "The name of the pod from which the data originated.", "required": false}, "peer_addr": {"description": "The IP from which the data originated.", "required": false}, "peer_path": {"description": "The pathname from which the data originated.", "required": false}, "mode": "_mode"}})))] ComponentReceivedEventsTotal, /// The number of event bytes accepted by this component either from @@ -41,11 +54,11 @@ pub enum CounterName { ComponentSentEventBytesTotal, /// The number of raw bytes sent by this component to destination sinks. - #[configurable(metadata(docs::tags = COMPONENT_SENT_BYTES_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"endpoint": {"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false}, "file": {"description": "The absolute path of the destination file.", "required": false}, "protocol": {"description": "The protocol used to send the bytes.", "required": true}, "region": {"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false}}})))] ComponentSentBytesTotal, /// The number of events dropped by this component. - #[configurable(metadata(docs::tags = COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"intentional": {"description": "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", "required": true}}})))] ComponentDiscardedEventsTotal, /// The total number of errors encountered by this component. @@ -268,7 +281,7 @@ pub enum CounterName { TagCardinalityUntrackedEventsTotal, /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. - #[configurable(metadata(docs::tags = TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"metric_name": {"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false}, "tag_key": {"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false}}})))] TagValueLimitExceededTotal, /// The total number of times the value limit was reached. @@ -313,7 +326,7 @@ pub enum CounterName { K8sDockerFormatParseFailuresTotal, /// The total number of times an S3 record in an SQS message was ignored. - #[configurable(metadata(docs::tags = SQS_S3_IGNORED_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"ignore_type": {"description": "The reason for ignoring the S3 record", "required": true, "enum": {"invalid_event_kind": "The kind of invalid event."}}}})))] SqsS3EventRecordIgnoredTotal, /// The total number of bytes allocated by this component. @@ -341,11 +354,11 @@ pub enum CounterName { MemoryEnrichmentTableTtlExpirations, /// The total number of errors reading datagram. - #[configurable(metadata(docs::tags = CONNECTION_READ_ERRORS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"mode": {"description": "", "required": true, "enum": {"udp": "User Datagram Protocol"}}}})))] ConnectionReadErrorsTotal, /// The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`. - #[configurable(metadata(docs::tags = EMPTY_TAGS))] + #[configurable(metadata(docs::tags = "{}"))] InternalMetricsCardinalityTotal, /// Number of configuration reload attempts that were rejected. @@ -353,7 +366,7 @@ pub enum CounterName { ConfigReloadRejected, /// The total number of errors converting bytes to a UTF-8 string in UDP mode. - #[configurable(metadata(docs::tags = UTF8_CONVERT_ERRORS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"mode": {"description": "The connection mode used by the component.", "required": true, "enum": {"udp": "User Datagram Protocol"}}}})))] Utf8ConvertErrorsTotal, } @@ -415,11 +428,11 @@ pub enum HistogramName { AdaptiveConcurrencyReachedLimit, /// The time taken to process an S3 object that succeeded, in seconds. - #[configurable(metadata(docs::tags = S3_OBJECT_PROCESSING_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"bucket": {"description": "The name of the S3 bucket.", "required": true}}})))] S3ObjectProcessingSucceededDurationSeconds, /// The time taken to process an S3 object that failed, in seconds. - #[configurable(metadata(docs::tags = S3_OBJECT_PROCESSING_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"bucket": {"description": "The name of the S3 bucket.", "required": true}}})))] S3ObjectProcessingFailedDurationSeconds, /// The duration spent collecting metrics for this component. @@ -599,7 +612,7 @@ pub enum GaugeName { UptimeSeconds, /// Pseudo-metric that provides build information for the Vector instance. - #[configurable(metadata(docs::tags = BUILD_INFO_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_internal_metrics_tags", "extra": {"debug": {"description": "Whether this is a debug build of Vector", "required": true}, "version": {"description": "Vector version.", "required": true}, "rust_version": {"description": "The Rust version from the package manifest.", "required": true}, "arch": {"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true}, "revision": {"description": "Revision identifer, related to versioned releases.", "required": true}}})))] BuildInfo, /// Current number of messages in producer queues. @@ -609,7 +622,7 @@ pub enum GaugeName { KafkaQueueMessagesBytes, /// The Kafka consumer lag. - #[configurable(metadata(docs::tags = KAFKA_CONSUMER_LAG_TAGS))] + #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"topic_id": {"description": "The Kafka topic id.", "required": true}, "partition_id": {"description": "The Kafka partition id.", "required": true}}})))] KafkaConsumerLag, /// The total memory currently being used by the Lua runtime. @@ -639,7 +652,7 @@ pub enum GaugeName { TagCardinalityTrackedKeys, /// The total number of metrics emitted from the internal metrics registry. - #[configurable(metadata(docs::tags = EMPTY_TAGS))] + #[configurable(metadata(docs::tags = "{}"))] InternalMetricsCardinality, } diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs deleted file mode 100644 index 9f872ae866d73..0000000000000 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ /dev/null @@ -1,431 +0,0 @@ -use serde::Serialize; - -/// A complete tag set for a Vector internal metric, represented as structured -/// data that is serialized to a CUE expression at schema-generation time. -/// -/// Use the pre-defined constants (`COMPONENT_TAGS`, `COMPONENT_TAGS_OUTPUT`, -/// etc.) or construct one inline for metric-specific tag shapes. -#[derive(Clone, Copy)] -pub struct TagSet { - /// Base CUE tag group (e.g. `"_component_tags"`). `None` means an empty - /// base (`{}`). - pub base: Option<&'static str>, - /// Extra fields merged into the base with `&`. - pub extra: &'static [ExtraTag], -} - -/// A single extra tag field appended to a [`TagSet`]. -#[derive(Clone, Copy)] -pub struct ExtraTag { - pub name: &'static str, - pub field: TagField, -} - -/// Definition of one tag field. -#[derive(Clone, Copy)] -pub enum TagField { - /// Reference to an existing CUE tag helper variable (e.g. `_output`). - Ref(&'static str), - /// Inline field definition with an optional fixed-value enum. - Inline { - description: &'static str, - required: bool, - enum_values: Option<&'static [(&'static str, &'static str)]>, - }, -} - -impl TagSet { - fn format_extras(self) -> Option { - if self.extra.is_empty() { - return None; - } - let parts: Vec = self - .extra - .iter() - .map(|et| { - let field = match et.field { - TagField::Ref(r) => r.to_owned(), - TagField::Inline { - description, - required, - enum_values: None, - } => format!(r#"{{description: "{description}", required: {required}}}"#), - TagField::Inline { - description, - required, - enum_values: Some(enums), - } => { - let pairs: Vec = - enums.iter().map(|(k, v)| format!("{k}: \"{v}\"")).collect(); - format!( - r#"{{description: "{description}", required: {required}, enum: {{{}}}}}"#, - pairs.join(", ") - ) - } - }; - format!("{}: {field}", et.name) - }) - .collect(); - Some(parts.join(", ")) - } - - /// Renders the tag set as a CUE expression (emitted verbatim by vdev). - #[must_use] - pub fn to_cue(self) -> String { - match (self.base, self.format_extras().as_deref()) { - (None, None) => "{}".to_owned(), - (Some(base), None) => base.to_owned(), - (None, Some(ex)) => format!("{{{ex}}}"), - (Some(base), Some(ex)) => format!("{base} & {{{ex}}}"), - } - } -} - -impl Serialize for TagSet { - fn serialize(&self, s: S) -> Result { - s.serialize_str(&self.to_cue()) - } -} - -// ─── Common base tag sets ────────────────────────────────────────────────────── - -pub const COMPONENT_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[], -}; - -pub const INTERNAL_METRICS_TAGS: TagSet = TagSet { - base: Some("_internal_metrics_tags"), - extra: &[], -}; - -pub const EMPTY_TAGS: TagSet = TagSet { - base: None, - extra: &[], -}; - -// ─── Cross-reference ────────────────────────────────────────────────────────── - -/// Inherits the tag set defined on `component_received_events_total`. -pub const COMPONENT_RECEIVED_EVENTS_TAGS: TagSet = TagSet { - base: Some("component_received_events_total.tags"), - extra: &[], -}; - -// ─── Component tags with a single CUE-helper extra field ────────────────────── - -pub const COMPONENT_TAGS_OUTPUT: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ExtraTag { name: "output", field: TagField::Ref("_output") }], -}; - -pub const INTERNAL_METRICS_TAGS_FILE: TagSet = TagSet { - base: Some("_internal_metrics_tags"), - extra: &[ExtraTag { name: "file", field: TagField::Ref("_file") }], -}; - -pub const INTERNAL_METRICS_TAGS_REASON: TagSet = TagSet { - base: Some("_internal_metrics_tags"), - extra: &[ExtraTag { name: "reason", field: TagField::Ref("_reason") }], -}; - -pub const COMPONENT_TAGS_ERROR_TYPE_STAGE: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ - ExtraTag { name: "error_type", field: TagField::Ref("_error_type") }, - ExtraTag { name: "stage", field: TagField::Ref("_stage") }, - ], -}; - -// ─── gRPC tag sets ──────────────────────────────────────────────────────────── - -pub const COMPONENT_TAGS_GRPC_METHOD_SERVICE: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ - ExtraTag { name: "grpc_method", field: TagField::Ref("_grpc_method") }, - ExtraTag { name: "grpc_service", field: TagField::Ref("_grpc_service") }, - ], -}; - -pub const COMPONENT_TAGS_GRPC_ALL: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ - ExtraTag { name: "grpc_method", field: TagField::Ref("_grpc_method") }, - ExtraTag { name: "grpc_service", field: TagField::Ref("_grpc_service") }, - ExtraTag { name: "grpc_status", field: TagField::Ref("_grpc_status") }, - ], -}; - -// ─── HTTP tag sets ──────────────────────────────────────────────────────────── - -pub const COMPONENT_TAGS_HTTP_METHOD: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ExtraTag { name: "method", field: TagField::Ref("_method") }], -}; - -pub const COMPONENT_TAGS_HTTP_STATUS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ExtraTag { name: "status", field: TagField::Ref("_status") }], -}; - -pub const COMPONENT_TAGS_HTTP_METHOD_PATH: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ - ExtraTag { name: "method", field: TagField::Ref("_method") }, - ExtraTag { name: "path", field: TagField::Ref("_path") }, - ], -}; - -pub const COMPONENT_TAGS_HTTP_ALL: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ - ExtraTag { name: "method", field: TagField::Ref("_method") }, - ExtraTag { name: "path", field: TagField::Ref("_path") }, - ExtraTag { name: "status", field: TagField::Ref("_status") }, - ], -}; - -// ─── Metric-specific tag sets ───────────────────────────────────────────────── - -pub const COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ - ExtraTag { - name: "file", - field: TagField::Inline { - description: "The file from which the data originated.", - required: false, - enum_values: None, - }, - }, - ExtraTag { - name: "uri", - field: TagField::Inline { - description: "The sanitized URI from which the data originated.", - required: false, - enum_values: None, - }, - }, - ExtraTag { - name: "container_name", - field: TagField::Inline { - description: "The name of the container from which the data originated.", - required: false, - enum_values: None, - }, - }, - ExtraTag { - name: "pod_name", - field: TagField::Inline { - description: "The name of the pod from which the data originated.", - required: false, - enum_values: None, - }, - }, - ExtraTag { - name: "peer_addr", - field: TagField::Inline { - description: "The IP from which the data originated.", - required: false, - enum_values: None, - }, - }, - ExtraTag { - name: "peer_path", - field: TagField::Inline { - description: "The pathname from which the data originated.", - required: false, - enum_values: None, - }, - }, - ExtraTag { name: "mode", field: TagField::Ref("_mode") }, - ], -}; - -pub const COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ExtraTag { - name: "intentional", - field: TagField::Inline { - description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", - required: true, - enum_values: None, - }, - }], -}; - -pub const COMPONENT_SENT_BYTES_TOTAL_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ - ExtraTag { - name: "endpoint", - field: TagField::Inline { - description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", - required: false, - enum_values: None, - }, - }, - ExtraTag { - name: "file", - field: TagField::Inline { - description: "The absolute path of the destination file.", - required: false, - enum_values: None, - }, - }, - ExtraTag { - name: "protocol", - field: TagField::Inline { - description: "The protocol used to send the bytes.", - required: true, - enum_values: None, - }, - }, - ExtraTag { - name: "region", - field: TagField::Inline { - description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", - required: false, - enum_values: None, - }, - }, - ], -}; - -pub const S3_OBJECT_PROCESSING_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ExtraTag { - name: "bucket", - field: TagField::Inline { - description: "The name of the S3 bucket.", - required: true, - enum_values: None, - }, - }], -}; - -pub const KAFKA_CONSUMER_LAG_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ - ExtraTag { - name: "topic_id", - field: TagField::Inline { - description: "The Kafka topic id.", - required: true, - enum_values: None, - }, - }, - ExtraTag { - name: "partition_id", - field: TagField::Inline { - description: "The Kafka partition id.", - required: true, - enum_values: None, - }, - }, - ], -}; - -pub const TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ - ExtraTag { - name: "metric_name", - field: TagField::Inline { - description: "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", - required: false, - enum_values: None, - }, - }, - ExtraTag { - name: "tag_key", - field: TagField::Inline { - description: "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", - required: false, - enum_values: None, - }, - }, - ], -}; - -pub const SQS_S3_IGNORED_TOTAL_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ExtraTag { - name: "ignore_type", - field: TagField::Inline { - description: "The reason for ignoring the S3 record", - required: true, - enum_values: Some(&[("invalid_event_kind", "The kind of invalid event.")]), - }, - }], -}; - -pub const BUILD_INFO_TAGS: TagSet = TagSet { - base: Some("_internal_metrics_tags"), - extra: &[ - ExtraTag { - name: "debug", - field: TagField::Inline { - description: "Whether this is a debug build of Vector", - required: true, - enum_values: None, - }, - }, - ExtraTag { - name: "version", - field: TagField::Inline { - description: "Vector version.", - required: true, - enum_values: None, - }, - }, - ExtraTag { - name: "rust_version", - field: TagField::Inline { - description: "The Rust version from the package manifest.", - required: true, - enum_values: None, - }, - }, - ExtraTag { - name: "arch", - field: TagField::Inline { - description: "The target architecture being compiled for. (e.g. x86_64)", - required: true, - enum_values: None, - }, - }, - ExtraTag { - name: "revision", - field: TagField::Inline { - description: "Revision identifer, related to versioned releases.", - required: true, - enum_values: None, - }, - }, - ], -}; - -pub const CONNECTION_READ_ERRORS_TOTAL_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ExtraTag { - name: "mode", - field: TagField::Inline { - description: "", - required: true, - enum_values: Some(&[("udp", "User Datagram Protocol")]), - }, - }], -}; - -pub const UTF8_CONVERT_ERRORS_TOTAL_TAGS: TagSet = TagSet { - base: Some("_component_tags"), - extra: &[ExtraTag { - name: "mode", - field: TagField::Inline { - description: "The connection mode used by the component.", - required: true, - enum_values: Some(&[("udp", "User Datagram Protocol")]), - }, - }], -}; diff --git a/lib/vector-common/src/internal_event/mod.rs b/lib/vector-common/src/internal_event/mod.rs index 822299ccf19e2..e776e89553172 100644 --- a/lib/vector-common/src/internal_event/mod.rs +++ b/lib/vector-common/src/internal_event/mod.rs @@ -6,7 +6,6 @@ pub mod component_events_timed_out; mod events_received; mod events_sent; pub mod metric_name; -pub mod metric_tags; mod optional_tag; mod prelude; pub mod service; diff --git a/lib/vector-config-macros/src/ast/mod.rs b/lib/vector-config-macros/src/ast/mod.rs index f997f3c68a8da..6b6a14f241fef 100644 --- a/lib/vector-config-macros/src/ast/mod.rs +++ b/lib/vector-config-macros/src/ast/mod.rs @@ -237,6 +237,11 @@ impl FromMeta for Metadata { key: path_to_string(&nv.path), value: path.to_token_stream(), }), + // Accept macro invocations such as `serde_json::json!({...})`. + Expr::Macro(mac) => Some(LazyCustomAttribute::KeyValue { + key: path_to_string(&nv.path), + value: mac.to_token_stream(), + }), expr => { errors .push(darling::Error::unexpected_expr_type(expr).with_span(nmeta)); diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index a837ba0294d9c..c8c52c796272e 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -362,6 +362,74 @@ fn render_and_import_generated_top_level_config_schema( Ok(()) } +/// Converts a `docs::tags` JSON object produced by `serde_json::json!()` to a CUE expression. +/// +/// Expected shape: +/// ```json +/// { +/// "base": "_component_tags", // optional CUE tag-group reference +/// "extra": { +/// "output": "_output", // string value → CUE reference +/// "mode": { // object value → inline definition +/// "description": "...", +/// "required": true, +/// "enum": { "udp": "..." } // optional +/// } +/// } +/// } +/// ``` +fn json_tags_to_cue(obj: &serde_json::Map) -> String { + let base = obj.get("base").and_then(Value::as_str); + + let extras = obj.get("extra").and_then(Value::as_object).map(|extra| { + extra + .iter() + .map(|(name, val)| match val { + Value::String(cue_ref) => format!("{name}: {cue_ref}"), + Value::Object(field) => { + let desc = field + .get("description") + .and_then(Value::as_str) + .unwrap_or(""); + let required = field + .get("required") + .and_then(Value::as_bool) + .unwrap_or(false); + let enum_part = + field + .get("enum") + .and_then(Value::as_object) + .map_or(String::new(), |e| { + let pairs: Vec = e + .iter() + .map(|(k, v)| { + format!( + "{k}: \"{}\"", + v.as_str().unwrap_or_default() + ) + }) + .collect(); + format!(", enum: {{{}}}", pairs.join(", ")) + }); + format!( + "{name}: {{description: \"{desc}\", required: {required}{enum_part}}}" + ) + } + _ => String::new(), + }) + .filter(|s| !s.is_empty()) + .collect::>() + .join(", ") + }); + + match (base, extras.as_deref()) { + (None, None | Some("")) => "{}".to_owned(), + (Some(b), None | Some("")) => b.to_owned(), + (None, Some(ex)) => format!("{{{ex}}}"), + (Some(b), Some(ex)) => format!("{b} & {{{ex}}}"), + } +} + struct MetricEntry { name: String, metric_type: &'static str, @@ -409,12 +477,11 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { .and_then(|m| m.get("deprecated_message")) .and_then(Value::as_str) .map(str::to_owned); - let tags = variant - .get("_metadata") - .and_then(|m| m.get("docs::tags")) - .and_then(Value::as_str) - .unwrap_or("_component_tags") - .to_owned(); + let tags = match variant.get("_metadata").and_then(|m| m.get("docs::tags")) { + Some(Value::String(s)) => s.clone(), + Some(Value::Object(obj)) => json_tags_to_cue(obj), + _ => "_component_tags".to_owned(), + }; entries.push(MetricEntry { name: name.to_owned(), From a8cd2c35ec0e2fe01c28667f3deb984937318b15 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 13:26:32 -0400 Subject: [PATCH 21/43] chore(metrics): define all tag groups in Rust using std::sync::LazyLock, remove CUE tag helpers --- .../src/internal_event/metric_name.rs | 75 ++- .../src/internal_event/metric_tags.rs | 451 ++++++++++++++++++ lib/vector-common/src/internal_event/mod.rs | 1 + .../commands/build/component_docs/runner.rs | 114 ++--- .../components/sources/internal_metrics.cue | 145 ------ .../internal_metric_descriptions.cue | 138 +++--- 6 files changed, 609 insertions(+), 315 deletions(-) create mode 100644 lib/vector-common/src/internal_event/metric_tags.rs diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index a007c5c5c1973..45b911d735fdc 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -1,29 +1,16 @@ -use serde_json::json; use strum::{AsRefStr, Display, EnumIter}; use vector_config::configurable_component; -// CUE tag-set constants for patterns shared across multiple metrics. -// Values are verbatim CUE expressions emitted by `vdev build component-docs`. -// Inline `json!()` is used directly on variants whose tag shape is unique. -pub const COMPONENT_TAGS: &str = "_component_tags"; -pub const INTERNAL_METRICS_TAGS: &str = "_internal_metrics_tags"; -pub const COMPONENT_TAGS_OUTPUT: &str = "_component_tags & {output: _output}"; -pub const INTERNAL_METRICS_TAGS_FILE: &str = "_internal_metrics_tags & {file: _file}"; -pub const INTERNAL_METRICS_TAGS_REASON: &str = "_internal_metrics_tags & {reason: _reason}"; -pub const COMPONENT_TAGS_GRPC_METHOD_SERVICE: &str = - "_component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service}"; -pub const COMPONENT_TAGS_GRPC_ALL: &str = - "_component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service, grpc_status: _grpc_status}"; -pub const COMPONENT_TAGS_HTTP_METHOD: &str = "_component_tags & {method: _method}"; -pub const COMPONENT_TAGS_HTTP_STATUS: &str = "_component_tags & {status: _status}"; -pub const COMPONENT_TAGS_HTTP_METHOD_PATH: &str = - "_component_tags & {method: _method, path: _path}"; -pub const COMPONENT_TAGS_HTTP_ALL: &str = - "_component_tags & {method: _method, path: _path, status: _status}"; -pub const COMPONENT_TAGS_ERROR_TYPE_STAGE: &str = - "_component_tags & {error_type: _error_type, stage: _stage}"; -/// Inherits the tag set defined on `component_received_events_total`. -pub const COMPONENT_RECEIVED_EVENTS_TAGS: &str = "component_received_events_total.tags"; +use super::metric_tags::{ + BUILD_INFO_TAGS, COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS, COMPONENT_RECEIVED_EVENTS_TAGS, + COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS, COMPONENT_SENT_BYTES_TOTAL_TAGS, + COMPONENT_TAGS_ERROR_TYPE_STAGE, COMPONENT_TAGS_GRPC_ALL, COMPONENT_TAGS_GRPC_METHOD_SERVICE, + COMPONENT_TAGS_HTTP_ALL, COMPONENT_TAGS_HTTP_METHOD, COMPONENT_TAGS_HTTP_METHOD_PATH, + COMPONENT_TAGS_HTTP_STATUS, COMPONENT_TAGS_OUTPUT, CONNECTION_READ_ERRORS_TOTAL_TAGS, + INTERNAL_METRICS_TAGS, INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, + KAFKA_CONSUMER_LAG_TAGS, S3_OBJECT_PROCESSING_TAGS, SQS_S3_IGNORED_TOTAL_TAGS, + TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS, UTF8_CONVERT_ERRORS_TOTAL_TAGS, +}; /// Canonical list of all per-component internal counter metric names emitted by Vector. #[configurable_component] @@ -33,7 +20,7 @@ pub const COMPONENT_RECEIVED_EVENTS_TAGS: &str = "component_received_events_tota pub enum CounterName { /// The number of events accepted by this component either from tagged /// origins like file and uri, or cumulatively from other origins. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"file": {"description": "The file from which the data originated.", "required": false}, "uri": {"description": "The sanitized URI from which the data originated.", "required": false}, "container_name": {"description": "The name of the container from which the data originated.", "required": false}, "pod_name": {"description": "The name of the pod from which the data originated.", "required": false}, "peer_addr": {"description": "The IP from which the data originated.", "required": false}, "peer_path": {"description": "The pathname from which the data originated.", "required": false}, "mode": "_mode"}})))] + #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS))] ComponentReceivedEventsTotal, /// The number of event bytes accepted by this component either from @@ -54,11 +41,11 @@ pub enum CounterName { ComponentSentEventBytesTotal, /// The number of raw bytes sent by this component to destination sinks. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"endpoint": {"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false}, "file": {"description": "The absolute path of the destination file.", "required": false}, "protocol": {"description": "The protocol used to send the bytes.", "required": true}, "region": {"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false}}})))] + #[configurable(metadata(docs::tags = COMPONENT_SENT_BYTES_TOTAL_TAGS))] ComponentSentBytesTotal, /// The number of events dropped by this component. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"intentional": {"description": "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", "required": true}}})))] + #[configurable(metadata(docs::tags = COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS))] ComponentDiscardedEventsTotal, /// The total number of errors encountered by this component. @@ -281,7 +268,7 @@ pub enum CounterName { TagCardinalityUntrackedEventsTotal, /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"metric_name": {"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false}, "tag_key": {"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false}}})))] + #[configurable(metadata(docs::tags = TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS))] TagValueLimitExceededTotal, /// The total number of times the value limit was reached. @@ -326,7 +313,7 @@ pub enum CounterName { K8sDockerFormatParseFailuresTotal, /// The total number of times an S3 record in an SQS message was ignored. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"ignore_type": {"description": "The reason for ignoring the S3 record", "required": true, "enum": {"invalid_event_kind": "The kind of invalid event."}}}})))] + #[configurable(metadata(docs::tags = SQS_S3_IGNORED_TOTAL_TAGS))] SqsS3EventRecordIgnoredTotal, /// The total number of bytes allocated by this component. @@ -354,7 +341,7 @@ pub enum CounterName { MemoryEnrichmentTableTtlExpirations, /// The total number of errors reading datagram. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"mode": {"description": "", "required": true, "enum": {"udp": "User Datagram Protocol"}}}})))] + #[configurable(metadata(docs::tags = CONNECTION_READ_ERRORS_TOTAL_TAGS))] ConnectionReadErrorsTotal, /// The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`. @@ -366,7 +353,7 @@ pub enum CounterName { ConfigReloadRejected, /// The total number of errors converting bytes to a UTF-8 string in UDP mode. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"mode": {"description": "The connection mode used by the component.", "required": true, "enum": {"udp": "User Datagram Protocol"}}}})))] + #[configurable(metadata(docs::tags = UTF8_CONVERT_ERRORS_TOTAL_TAGS))] Utf8ConvertErrorsTotal, } @@ -428,11 +415,11 @@ pub enum HistogramName { AdaptiveConcurrencyReachedLimit, /// The time taken to process an S3 object that succeeded, in seconds. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"bucket": {"description": "The name of the S3 bucket.", "required": true}}})))] + #[configurable(metadata(docs::tags = S3_OBJECT_PROCESSING_TAGS))] S3ObjectProcessingSucceededDurationSeconds, /// The time taken to process an S3 object that failed, in seconds. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"bucket": {"description": "The name of the S3 bucket.", "required": true}}})))] + #[configurable(metadata(docs::tags = S3_OBJECT_PROCESSING_TAGS))] S3ObjectProcessingFailedDurationSeconds, /// The duration spent collecting metrics for this component. @@ -521,12 +508,16 @@ pub enum GaugeName { ComponentLatencyMeanSeconds, /// The maximum number of events the source buffer can hold. - #[configurable(deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_events`.")] + #[configurable( + deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_events`." + )] #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] SourceBufferMaxEventSize, /// The maximum number of bytes the source buffer can hold. - #[configurable(deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_bytes`.")] + #[configurable( + deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_bytes`." + )] #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] SourceBufferMaxByteSize, @@ -547,12 +538,16 @@ pub enum GaugeName { SourceBufferUtilizationMean, /// The maximum number of events the buffer that feeds into a transform can hold. - #[configurable(deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_events`.")] + #[configurable( + deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_events`." + )] #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] TransformBufferMaxEventSize, /// The maximum number of bytes the buffer that feeds into a transform can hold. - #[configurable(deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_bytes`.")] + #[configurable( + deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_bytes`." + )] #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] TransformBufferMaxByteSize, @@ -585,7 +580,9 @@ pub enum GaugeName { BufferMaxByteSize, /// The number of events currently in the buffer. - #[configurable(deprecated = "This metric has been deprecated in favor of `buffer_size_events`.")] + #[configurable( + deprecated = "This metric has been deprecated in favor of `buffer_size_events`." + )] BufferEvents, /// The number of events currently in the buffer. @@ -612,7 +609,7 @@ pub enum GaugeName { UptimeSeconds, /// Pseudo-metric that provides build information for the Vector instance. - #[configurable(metadata(docs::tags = json!({"base": "_internal_metrics_tags", "extra": {"debug": {"description": "Whether this is a debug build of Vector", "required": true}, "version": {"description": "Vector version.", "required": true}, "rust_version": {"description": "The Rust version from the package manifest.", "required": true}, "arch": {"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true}, "revision": {"description": "Revision identifer, related to versioned releases.", "required": true}}})))] + #[configurable(metadata(docs::tags = BUILD_INFO_TAGS))] BuildInfo, /// Current number of messages in producer queues. @@ -622,7 +619,7 @@ pub enum GaugeName { KafkaQueueMessagesBytes, /// The Kafka consumer lag. - #[configurable(metadata(docs::tags = json!({"base": "_component_tags", "extra": {"topic_id": {"description": "The Kafka topic id.", "required": true}, "partition_id": {"description": "The Kafka partition id.", "required": true}}})))] + #[configurable(metadata(docs::tags = KAFKA_CONSUMER_LAG_TAGS))] KafkaConsumerLag, /// The total memory currently being used by the Lua runtime. diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs new file mode 100644 index 0000000000000..ce2f01726b9d3 --- /dev/null +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -0,0 +1,451 @@ +use serde_json::{Value, json}; +use std::sync::LazyLock; + +/// A compile-time handle to a lazily-initialized JSON tag-set. +/// +/// Holds a function pointer so it can be used as a `const` in +/// `#[configurable(metadata(docs::tags = MY_TAGS))]` attributes. +pub struct TagSet(pub fn() -> &'static Value); + +impl serde::Serialize for TagSet { + fn serialize(&self, s: S) -> Result { + (self.0)().serialize(s) + } +} + +// ─── Individual tag field definitions ───────────────────────────────────────── + +fn field_pid() -> Value { + json!({"description": "The process ID of the Vector instance.", "required": false}) +} +fn field_host() -> Value { + json!({"description": "The hostname of the system Vector is running on.", "required": false}) +} +fn field_component_kind() -> Value { + json!({"description": "The Vector component kind.", "required": true, "enum": { + "sink": "Vector sink components", + "source": "Vector source components", + "transform": "Vector transform components" + }}) +} +fn field_component_id() -> Value { + json!({"description": "The Vector component ID.", "required": true}) +} +fn field_component_type() -> Value { + json!({"description": "The Vector component type.", "required": true}) +} +fn field_output() -> Value { + json!({"description": "The specific output of the component.", "required": false}) +} +fn field_mode_transport() -> Value { + json!({"description": "The connection mode used by the component.", "required": false, "enum": { + "udp": "User Datagram Protocol", + "tcp": "Transmission Control Protocol", + "unix": "Unix domain socket" + }}) +} +fn field_grpc_method() -> Value { + json!({"description": "The name of the method called on the gRPC service.", "required": true}) +} +fn field_grpc_service() -> Value { + json!({"description": "The gRPC service name.", "required": true}) +} +fn field_grpc_status() -> Value { + json!({"description": "The human-readable gRPC status code.", "required": true}) +} +fn field_http_status() -> Value { + json!({"description": "The HTTP status code of the request.", "required": false}) +} +fn field_http_method() -> Value { + json!({"description": "The HTTP method of the request.", "required": false}) +} +fn field_http_path() -> Value { + json!({"description": "The path that produced the error.", "required": true}) +} +fn field_file() -> Value { + json!({"description": "The file that produced the error.", "required": false}) +} +fn field_reason() -> Value { + json!({"description": "The type of the error", "required": true, "enum": { + "out_of_order": "The event was out of order.", + "oversized": "The event was too large." + }}) +} +fn field_error_type() -> Value { + json!({"description": "The type of the error", "required": true, "enum": { + "acknowledgements_failed": "The acknowledgement operation failed.", + "delete_failed": "The file deletion failed.", + "encode_failed": "The encode operation failed.", + "field_missing": "The event field was missing.", + "glob_failed": "The glob pattern match operation failed.", + "http_error": "The HTTP request resulted in an error code.", + "invalid_metric": "The metric was invalid.", + "kafka_offset_update": "The consumer offset update failed.", + "kafka_read": "The message from Kafka was invalid.", + "mapping_failed": "The mapping failed.", + "match_failed": "The match operation failed.", + "out_of_order": "The event was out of order.", + "parse_failed": "The parsing operation failed.", + "read_failed": "The file read operation failed.", + "render_error": "The rendering operation failed.", + "stream_closed": "The downstream was closed, forwarding the event(s) failed.", + "type_conversion_failed": "The type conversion operating failed.", + "type_field_does_not_exist": "The type field does not exist.", + "type_ip_address_parse_error": "The IP address did not parse.", + "unlabeled_event": "The event was not labeled.", + "value_invalid": "The value was invalid.", + "watch_failed": "The file watch operation failed.", + "write_failed": "The file write operation failed." + }}) +} +fn field_stage() -> Value { + json!({"description": "The stage within the component at which the error occurred.", "required": true, "enum": { + "receiving": "While receiving data.", + "processing": "While processing data within the component.", + "sending": "While sending data." + }}) +} + +// ─── Base tag groups ─────────────────────────────────────────────────────────── + +fn internal_metrics_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| json!({"pid": field_pid(), "host": field_host()})); + &V +} + +fn component_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = internal_metrics_tags_value().clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("component_kind".to_owned(), field_component_kind()); + obj.insert("component_id".to_owned(), field_component_id()); + obj.insert("component_type".to_owned(), field_component_type()); + tags + }); + &V +} + +fn empty_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| json!({})); + &V +} + +// ─── Extensions of COMPONENT_TAGS ───────────────────────────────────────────── + +fn component_tags_output_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + tags.as_object_mut() + .unwrap() + .insert("output".to_owned(), field_output()); + tags + }); + &V +} + +fn component_tags_grpc_method_service_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("grpc_method".to_owned(), field_grpc_method()); + obj.insert("grpc_service".to_owned(), field_grpc_service()); + tags + }); + &V +} + +fn component_tags_grpc_all_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_grpc_method_service_value().clone(); + tags.as_object_mut() + .unwrap() + .insert("grpc_status".to_owned(), field_grpc_status()); + tags + }); + &V +} + +fn component_tags_http_method_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + tags.as_object_mut() + .unwrap() + .insert("method".to_owned(), field_http_method()); + tags + }); + &V +} + +fn component_tags_http_status_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + tags.as_object_mut() + .unwrap() + .insert("status".to_owned(), field_http_status()); + tags + }); + &V +} + +fn component_tags_http_method_path_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_http_method_value().clone(); + tags.as_object_mut() + .unwrap() + .insert("path".to_owned(), field_http_path()); + tags + }); + &V +} + +fn component_tags_http_all_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_http_method_path_value().clone(); + tags.as_object_mut() + .unwrap() + .insert("status".to_owned(), field_http_status()); + tags + }); + &V +} + +fn component_tags_error_type_stage_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("error_type".to_owned(), field_error_type()); + obj.insert("stage".to_owned(), field_stage()); + tags + }); + &V +} + +// ─── Extensions of INTERNAL_METRICS_TAGS ────────────────────────────────────── + +fn internal_metrics_tags_file_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = internal_metrics_tags_value().clone(); + tags.as_object_mut() + .unwrap() + .insert("file".to_owned(), field_file()); + tags + }); + &V +} + +fn internal_metrics_tags_reason_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = internal_metrics_tags_value().clone(); + tags.as_object_mut() + .unwrap() + .insert("reason".to_owned(), field_reason()); + tags + }); + &V +} + +// ─── Metric-specific tag sets ───────────────────────────────────────────────── + +fn component_received_events_total_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert( + "file".to_owned(), + json!({"description": "The file from which the data originated.", "required": false}), + ); + obj.insert("uri".to_owned(), + json!({"description": "The sanitized URI from which the data originated.", "required": false})); + obj.insert("container_name".to_owned(), + json!({"description": "The name of the container from which the data originated.", "required": false})); + obj.insert("pod_name".to_owned(), + json!({"description": "The name of the pod from which the data originated.", "required": false})); + obj.insert( + "peer_addr".to_owned(), + json!({"description": "The IP from which the data originated.", "required": false}), + ); + obj.insert("peer_path".to_owned(), + json!({"description": "The pathname from which the data originated.", "required": false})); + obj.insert("mode".to_owned(), field_mode_transport()); + tags + }); + &V +} + +fn component_discarded_events_total_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + tags.as_object_mut().unwrap().insert("intentional".to_owned(), json!({ + "description": "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", + "required": true + })); + tags + }); + &V +} + +fn component_sent_bytes_total_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("endpoint".to_owned(), json!({"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false})); + obj.insert( + "file".to_owned(), + json!({"description": "The absolute path of the destination file.", "required": false}), + ); + obj.insert( + "protocol".to_owned(), + json!({"description": "The protocol used to send the bytes.", "required": true}), + ); + obj.insert("region".to_owned(), json!({"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false})); + tags + }); + &V +} + +fn s3_object_processing_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + tags.as_object_mut().unwrap().insert( + "bucket".to_owned(), + json!({"description": "The name of the S3 bucket.", "required": true}), + ); + tags + }); + &V +} + +fn kafka_consumer_lag_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert( + "topic_id".to_owned(), + json!({"description": "The Kafka topic id.", "required": true}), + ); + obj.insert( + "partition_id".to_owned(), + json!({"description": "The Kafka partition id.", "required": true}), + ); + tags + }); + &V +} + +fn tag_value_limit_exceeded_total_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("metric_name".to_owned(), json!({"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); + obj.insert("tag_key".to_owned(), json!({"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); + tags + }); + &V +} + +fn sqs_s3_ignored_total_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + tags.as_object_mut().unwrap().insert( + "ignore_type".to_owned(), + json!({ + "description": "The reason for ignoring the S3 record", + "required": true, + "enum": {"invalid_event_kind": "The kind of invalid event."} + }), + ); + tags + }); + &V +} + +fn build_info_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = internal_metrics_tags_value().clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert( + "debug".to_owned(), + json!({"description": "Whether this is a debug build of Vector", "required": true}), + ); + obj.insert( + "version".to_owned(), + json!({"description": "Vector version.", "required": true}), + ); + obj.insert( + "rust_version".to_owned(), + json!({"description": "The Rust version from the package manifest.", "required": true}), + ); + obj.insert("arch".to_owned(), json!({"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true})); + obj.insert("revision".to_owned(), json!({"description": "Revision identifer, related to versioned releases.", "required": true})); + tags + }); + &V +} + +fn connection_read_errors_total_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + tags.as_object_mut().unwrap().insert( + "mode".to_owned(), + json!({ + "description": "", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + }), + ); + tags + }); + &V +} + +fn utf8_convert_errors_total_tags_value() -> &'static Value { + static V: LazyLock = LazyLock::new(|| { + let mut tags = component_tags_value().clone(); + tags.as_object_mut().unwrap().insert( + "mode".to_owned(), + json!({ + "description": "The connection mode used by the component.", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + }), + ); + tags + }); + &V +} + +// ─── Public constants ────────────────────────────────────────────────────────── + +pub const INTERNAL_METRICS_TAGS: TagSet = TagSet(internal_metrics_tags_value); +pub const COMPONENT_TAGS: TagSet = TagSet(component_tags_value); +pub const EMPTY_TAGS: TagSet = TagSet(empty_tags_value); + +/// Same tag set as `component_received_events_total` (inherited by byte-count metrics). +pub const COMPONENT_RECEIVED_EVENTS_TAGS: TagSet = + TagSet(component_received_events_total_tags_value); + +pub const COMPONENT_TAGS_OUTPUT: TagSet = TagSet(component_tags_output_value); +pub const COMPONENT_TAGS_GRPC_METHOD_SERVICE: TagSet = + TagSet(component_tags_grpc_method_service_value); +pub const COMPONENT_TAGS_GRPC_ALL: TagSet = TagSet(component_tags_grpc_all_value); +pub const COMPONENT_TAGS_HTTP_METHOD: TagSet = TagSet(component_tags_http_method_value); +pub const COMPONENT_TAGS_HTTP_STATUS: TagSet = TagSet(component_tags_http_status_value); +pub const COMPONENT_TAGS_HTTP_METHOD_PATH: TagSet = TagSet(component_tags_http_method_path_value); +pub const COMPONENT_TAGS_HTTP_ALL: TagSet = TagSet(component_tags_http_all_value); +pub const COMPONENT_TAGS_ERROR_TYPE_STAGE: TagSet = TagSet(component_tags_error_type_stage_value); +pub const INTERNAL_METRICS_TAGS_FILE: TagSet = TagSet(internal_metrics_tags_file_value); +pub const INTERNAL_METRICS_TAGS_REASON: TagSet = TagSet(internal_metrics_tags_reason_value); +pub const COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: TagSet = + TagSet(component_received_events_total_tags_value); +pub const COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS: TagSet = + TagSet(component_discarded_events_total_tags_value); +pub const COMPONENT_SENT_BYTES_TOTAL_TAGS: TagSet = TagSet(component_sent_bytes_total_tags_value); +pub const S3_OBJECT_PROCESSING_TAGS: TagSet = TagSet(s3_object_processing_tags_value); +pub const KAFKA_CONSUMER_LAG_TAGS: TagSet = TagSet(kafka_consumer_lag_tags_value); +pub const TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS: TagSet = + TagSet(tag_value_limit_exceeded_total_tags_value); +pub const SQS_S3_IGNORED_TOTAL_TAGS: TagSet = TagSet(sqs_s3_ignored_total_tags_value); +pub const BUILD_INFO_TAGS: TagSet = TagSet(build_info_tags_value); +pub const CONNECTION_READ_ERRORS_TOTAL_TAGS: TagSet = + TagSet(connection_read_errors_total_tags_value); +pub const UTF8_CONVERT_ERRORS_TOTAL_TAGS: TagSet = TagSet(utf8_convert_errors_total_tags_value); diff --git a/lib/vector-common/src/internal_event/mod.rs b/lib/vector-common/src/internal_event/mod.rs index e776e89553172..822299ccf19e2 100644 --- a/lib/vector-common/src/internal_event/mod.rs +++ b/lib/vector-common/src/internal_event/mod.rs @@ -6,6 +6,7 @@ pub mod component_events_timed_out; mod events_received; mod events_sent; pub mod metric_name; +pub mod metric_tags; mod optional_tag; mod prelude; pub mod service; diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index c8c52c796272e..4f3f0bf183f9b 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -362,71 +362,58 @@ fn render_and_import_generated_top_level_config_schema( Ok(()) } -/// Converts a `docs::tags` JSON object produced by `serde_json::json!()` to a CUE expression. +/// Converts a flat JSON tag-field object to a CUE struct literal. /// -/// Expected shape: +/// The JSON is a map of tag-field names to field definitions: /// ```json /// { -/// "base": "_component_tags", // optional CUE tag-group reference -/// "extra": { -/// "output": "_output", // string value → CUE reference -/// "mode": { // object value → inline definition -/// "description": "...", -/// "required": true, -/// "enum": { "udp": "..." } // optional -/// } -/// } +/// "component_id": {"description": "...", "required": true}, +/// "output": {"description": "...", "required": false}, +/// "mode": {"description": "...", "required": true, +/// "enum": {"udp": "User Datagram Protocol"}} /// } /// ``` +/// Produces: `{component_id: {description: "...", required: true}, output: {...}, ...}` fn json_tags_to_cue(obj: &serde_json::Map) -> String { - let base = obj.get("base").and_then(Value::as_str); + if obj.is_empty() { + return "{}".to_owned(); + } - let extras = obj.get("extra").and_then(Value::as_object).map(|extra| { - extra - .iter() - .map(|(name, val)| match val { - Value::String(cue_ref) => format!("{name}: {cue_ref}"), - Value::Object(field) => { - let desc = field - .get("description") - .and_then(Value::as_str) - .unwrap_or(""); - let required = field - .get("required") - .and_then(Value::as_bool) - .unwrap_or(false); - let enum_part = - field - .get("enum") - .and_then(Value::as_object) - .map_or(String::new(), |e| { - let pairs: Vec = e - .iter() - .map(|(k, v)| { - format!( - "{k}: \"{}\"", - v.as_str().unwrap_or_default() - ) - }) - .collect(); - format!(", enum: {{{}}}", pairs.join(", ")) - }); - format!( - "{name}: {{description: \"{desc}\", required: {required}{enum_part}}}" - ) - } - _ => String::new(), - }) - .filter(|s| !s.is_empty()) - .collect::>() - .join(", ") - }); + let fields: Vec = obj + .iter() + .map(|(name, field)| { + let field_cue = json_field_to_cue(field); + format!("{name}: {field_cue}") + }) + .collect(); - match (base, extras.as_deref()) { - (None, None | Some("")) => "{}".to_owned(), - (Some(b), None | Some("")) => b.to_owned(), - (None, Some(ex)) => format!("{{{ex}}}"), - (Some(b), Some(ex)) => format!("{b} & {{{ex}}}"), + format!("{{{}}}", fields.join(", ")) +} + +fn json_field_to_cue(val: &Value) -> String { + match val { + Value::Object(obj) => { + let desc = obj.get("description").and_then(Value::as_str).unwrap_or(""); + let required = obj + .get("required") + .and_then(Value::as_bool) + .unwrap_or(false); + let enum_part = obj + .get("enum") + .and_then(Value::as_object) + .map_or(String::new(), |e| { + let pairs: Vec = e + .iter() + .map(|(k, v)| format!("{k}: \"{}\"", v.as_str().unwrap_or_default())) + .collect(); + format!(", enum: {{{}}}", pairs.join(", ")) + }); + format!("{{description: \"{desc}\", required: {required}{enum_part}}}") + } + Value::String(s) => format!("\"{s}\""), + Value::Bool(b) => b.to_string(), + Value::Number(n) => n.to_string(), + _ => "{}".to_owned(), } } @@ -440,9 +427,8 @@ struct MetricEntry { } fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { - let out_path = PathBuf::from( - "website/cue/reference/generated/internal_metric_descriptions.cue", - ); + let out_path = + PathBuf::from("website/cue/reference/generated/internal_metric_descriptions.cue"); if let Some(parent) = out_path.parent() { fs::create_dir_all(parent)?; @@ -478,8 +464,9 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { .and_then(Value::as_str) .map(str::to_owned); let tags = match variant.get("_metadata").and_then(|m| m.get("docs::tags")) { - Some(Value::String(s)) => s.clone(), Some(Value::Object(obj)) => json_tags_to_cue(obj), + // String values are plain CUE expressions emitted verbatim (e.g. "{}"). + Some(Value::String(s)) => s.clone(), _ => "_component_tags".to_owned(), }; @@ -519,7 +506,10 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { if e.deprecated { cue.push_str("\t\tdeprecated: true\n"); if let Some(msg) = &e.deprecated_message { - let escaped = msg.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n"); + let escaped = msg + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n"); writeln!(cue, "\t\tdeprecated_message: \"{escaped}\"").unwrap(); } } diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index 38ea524d3bb58..7fa9479afd896 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -34,150 +34,5 @@ components: sources: internal_metrics: { configuration: generated.components.sources.internal_metrics.configuration - output: metrics: { - // Default internal metrics tags - _internal_metrics_tags: { - pid: { - description: "The process ID of the Vector instance." - required: false - examples: ["4232"] - } - host: { - description: "The hostname of the system Vector is running on." - required: false - examples: [_values.local_host] - } - } - - // Helpful tag groupings - _component_tags: _internal_metrics_tags & { - component_kind: _component_kind - component_id: _component_id - component_type: _component_type - } - - // All available tags - _collector: { - description: "Which collector this metric comes from." - required: true - } - _component_kind: { - description: "The Vector component kind." - required: true - enum: { - "sink": "Vector sink components" - "source": "Vector source components" - "transform": "Vector transform components" - } - } - _component_id: { - description: "The Vector component ID." - required: true - examples: ["my_source", "my_sink"] - } - _component_type: { - description: "The Vector component type." - required: true - examples: ["file", "http", "honeycomb", "splunk_hec"] - } - _endpoint: { - description: "The absolute path of originating file." - required: true - examples: ["http://localhost:8080/server-status?auto"] - } - _error_type: { - description: "The type of the error" - required: true - enum: { - "acknowledgements_failed": "The acknowledgement operation failed." - "delete_failed": "The file deletion failed." - "encode_failed": "The encode operation failed." - "field_missing": "The event field was missing." - "glob_failed": "The glob pattern match operation failed." - "http_error": "The HTTP request resulted in an error code." - "invalid_metric": "The metric was invalid." - "kafka_offset_update": "The consumer offset update failed." - "kafka_read": "The message from Kafka was invalid." - "mapping_failed": "The mapping failed." - "match_failed": "The match operation failed." - "out_of_order": "The event was out of order." - "parse_failed": "The parsing operation failed." - "read_failed": "The file read operation failed." - "render_error": "The rendering operation failed." - "stream_closed": "The downstream was closed, forwarding the event(s) failed." - "type_conversion_failed": "The type conversion operating failed." - "type_field_does_not_exist": "The type field does not exist." - "type_ip_address_parse_error": "The IP address did not parse." - "unlabeled_event": "The event was not labeled." - "value_invalid": "The value was invalid." - "watch_failed": "The file watch operation failed." - "write_failed": "The file write operation failed." - } - } - _file: { - description: "The file that produced the error" - required: false - } - _grpc_method: { - description: "The name of the method called on the gRPC service." - required: true - } - _grpc_service: { - description: "The gRPC service name." - required: true - } - _grpc_status: { - description: "The human-readable [gRPC status code](\(urls.grpc_status_code))." - required: true - } - _host: { - description: "The hostname of the originating system." - required: true - examples: [_values.local_host] - } - _mode: { - description: "The connection mode used by the component." - required: false - enum: { - udp: "User Datagram Protocol" - tcp: "Transmission Control Protocol" - unix: "Unix domain socket" - } - } - _output: { - description: "The specific output of the component." - required: false - } - _stage: { - description: "The stage within the component at which the error occurred." - required: true - enum: { - receiving: "While receiving data." - processing: "While processing data within the component." - sending: "While sending data." - } - } - _status: { - description: "The HTTP status code of the request." - required: false - } - _method: { - description: "The HTTP method of the request." - required: false - } - _path: { - description: "The path that produced the error." - required: true - } - _reason: { - description: "The type of the error" - required: true - enum: { - "out_of_order": "The event was out of order." - "oversized": "The event was too large." - } - } - } - how_it_works: {} } diff --git a/website/cue/reference/generated/internal_metric_descriptions.cue b/website/cue/reference/generated/internal_metric_descriptions.cue index 3ad7e27002a6c..136a0fca5f6cd 100644 --- a/website/cue/reference/generated/internal_metric_descriptions.cue +++ b/website/cue/reference/generated/internal_metric_descriptions.cue @@ -82,7 +82,7 @@ components: sources: internal_metrics: output: metrics: { description: "The number of times the Vector API has been started." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } buffer_byte_size: { description: "The number of bytes currently in the buffer." @@ -188,31 +188,31 @@ components: sources: internal_metrics: output: metrics: { description: "Pseudo-metric that provides build information for the Vector instance." type: "gauge" default_namespace: "vector" - tags: _internal_metrics_tags & {debug: {description: "Whether this is a debug build of Vector", required: true}, version: {description: "Vector version.", required: true}, rust_version: {description: "The Rust version from the package manifest.", required: true}, arch: {description: "The target architecture being compiled for. (e.g. x86_64)", required: true}, revision: {description: "Revision identifer, related to versioned releases.", required: true}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, debug: {description: "Whether this is a debug build of Vector", required: true}, version: {description: "Vector version.", required: true}, rust_version: {description: "The Rust version from the package manifest.", required: true}, arch: {description: "The target architecture being compiled for. (e.g. x86_64)", required: true}, revision: {description: "Revision identifer, related to versioned releases.", required: true}} } checkpoints_total: { description: "The total number of files checkpointed." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } checksum_errors_total: { description: "The total number of errors identifying files via checksum." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags & {file: _file} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} } collect_completed_total: { description: "The total number of metrics collections completed for this component." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } collect_duration_seconds: { description: "The duration spent collecting metrics for this component." type: "histogram" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } command_executed_total: { description: "The total number of times a command has been executed." @@ -248,73 +248,73 @@ components: sources: internal_metrics: output: metrics: { description: "The number of events dropped by this component." type: "counter" default_namespace: "vector" - tags: _component_tags & {intentional: {description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", required: true}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, intentional: {description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", required: true}} } component_errors_total: { description: "The total number of errors encountered by this component." type: "counter" default_namespace: "vector" - tags: _component_tags & {error_type: _error_type, stage: _stage} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, error_type: {description: "The type of the error", required: true, enum: {acknowledgements_failed: "The acknowledgement operation failed.", delete_failed: "The file deletion failed.", encode_failed: "The encode operation failed.", field_missing: "The event field was missing.", glob_failed: "The glob pattern match operation failed.", http_error: "The HTTP request resulted in an error code.", invalid_metric: "The metric was invalid.", kafka_offset_update: "The consumer offset update failed.", kafka_read: "The message from Kafka was invalid.", mapping_failed: "The mapping failed.", match_failed: "The match operation failed.", out_of_order: "The event was out of order.", parse_failed: "The parsing operation failed.", read_failed: "The file read operation failed.", render_error: "The rendering operation failed.", stream_closed: "The downstream was closed, forwarding the event(s) failed.", type_conversion_failed: "The type conversion operating failed.", type_field_does_not_exist: "The type field does not exist.", type_ip_address_parse_error: "The IP address did not parse.", unlabeled_event: "The event was not labeled.", value_invalid: "The value was invalid.", watch_failed: "The file watch operation failed.", write_failed: "The file write operation failed."}}, stage: {description: "The stage within the component at which the error occurred.", required: true, enum: {receiving: "While receiving data.", processing: "While processing data within the component.", sending: "While sending data."}}} } component_latency_mean_seconds: { description: "This includes both the time spent queued in the transform's input buffer and the time spent\nexecuting the transform itself. This value is smoothed over time using an exponentially\nweighted moving average (EWMA)." type: "gauge" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } component_latency_seconds: { description: "This includes both the time spent queued in the transform's input buffer and the time spent\nexecuting the transform itself." type: "histogram" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } component_received_bytes: { description: "The size in bytes of each event received by the source." type: "histogram" default_namespace: "vector" - tags: component_received_events_total.tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} } component_received_bytes_total: { description: "The number of raw bytes accepted by this component from source origins." type: "counter" default_namespace: "vector" - tags: component_received_events_total.tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} } component_received_event_bytes_total: { description: "The number of event bytes accepted by this component either from\ntagged origins like file and uri, or cumulatively from other origins." type: "counter" default_namespace: "vector" - tags: component_received_events_total.tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} } component_received_events_count: { description: "Note that this is separate than sink-level batching. It is mostly useful for low level\ndebugging performance issues in Vector due to small internal batches." type: "histogram" default_namespace: "vector" - tags: component_received_events_total.tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} } component_received_events_total: { description: "The number of events accepted by this component either from tagged\norigins like file and uri, or cumulatively from other origins." type: "counter" default_namespace: "vector" - tags: _component_tags & {file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: _mode} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} } component_sent_bytes_total: { description: "The number of raw bytes sent by this component to destination sinks." type: "counter" default_namespace: "vector" - tags: _component_tags & {endpoint: {description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", required: false}, file: {description: "The absolute path of the destination file.", required: false}, protocol: {description: "The protocol used to send the bytes.", required: true}, region: {description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", required: false}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, endpoint: {description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", required: false}, file: {description: "The absolute path of the destination file.", required: false}, protocol: {description: "The protocol used to send the bytes.", required: true}, region: {description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", required: false}} } component_sent_event_bytes_total: { description: "The total number of event bytes emitted by this component." type: "counter" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } component_sent_events_total: { description: "The total number of events emitted by this component." type: "counter" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } component_timed_out_events_total: { description: "The total number of events for which this source responded with a timeout error." @@ -332,31 +332,31 @@ components: sources: internal_metrics: output: metrics: { description: "Number of configuration reload attempts that were rejected." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags & {reason: _reason} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, reason: {description: "The type of the error", required: true, enum: {out_of_order: "The event was out of order.", oversized: "The event was too large."}}} } connection_established_total: { description: "The total number of times a connection has been established." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } connection_read_errors_total: { description: "The total number of errors reading datagram." type: "counter" default_namespace: "vector" - tags: _component_tags & {mode: {description: "", required: true, enum: {udp: "User Datagram Protocol"}}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, mode: {description: "", required: true, enum: {udp: "User Datagram Protocol"}}} } connection_send_errors_total: { description: "The total number of errors sending data via the connection." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } connection_shutdown_total: { description: "The total number of times the connection has been shut down." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } container_processed_events_total: { description: "The total number of container events processed." @@ -416,49 +416,49 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of events discarded by this component." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags & {reason: _reason} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, reason: {description: "The type of the error", required: true, enum: {out_of_order: "The event was out of order.", oversized: "The event was too large."}}} } files_added_total: { description: "The total number of files Vector has found to watch." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags & {file: _file} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} } files_deleted_total: { description: "The total number of files deleted." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags & {file: _file} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} } files_resumed_total: { description: "The total number of times Vector has resumed watching a file." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags & {file: _file} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} } files_unwatched_total: { description: "The total number of times Vector has stopped watching a file." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags & {file: _file} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} } grpc_server_handler_duration_seconds: { description: "The duration spent handling a gRPC request." type: "histogram" default_namespace: "vector" - tags: _component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service, grpc_status: _grpc_status} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, grpc_method: {description: "The name of the method called on the gRPC service.", required: true}, grpc_service: {description: "The gRPC service name.", required: true}, grpc_status: {description: "The human-readable gRPC status code.", required: true}} } grpc_server_messages_received_total: { description: "The total number of gRPC messages received." type: "counter" default_namespace: "vector" - tags: _component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, grpc_method: {description: "The name of the method called on the gRPC service.", required: true}, grpc_service: {description: "The gRPC service name.", required: true}} } grpc_server_messages_sent_total: { description: "The total number of gRPC messages sent." type: "counter" default_namespace: "vector" - tags: _component_tags & {grpc_method: _grpc_method, grpc_service: _grpc_service, grpc_status: _grpc_status} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, grpc_method: {description: "The name of the method called on the gRPC service.", required: true}, grpc_service: {description: "The gRPC service name.", required: true}, grpc_status: {description: "The human-readable gRPC status code.", required: true}} } http_client_error_rtt_seconds: { description: "The round-trip time (RTT) of HTTP requests that resulted in an error." @@ -476,19 +476,19 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of sent HTTP requests, tagged with the request method." type: "counter" default_namespace: "vector" - tags: _component_tags & {method: _method} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, method: {description: "The HTTP method of the request.", required: false}} } http_client_response_rtt_seconds: { description: "The round-trip time (RTT) of HTTP requests, tagged with the response code." type: "histogram" default_namespace: "vector" - tags: _component_tags & {status: _status} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, status: {description: "The HTTP status code of the request.", required: false}} } http_client_responses_total: { description: "The total number of HTTP requests, tagged with the response code." type: "counter" default_namespace: "vector" - tags: _component_tags & {status: _status} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, status: {description: "The HTTP status code of the request.", required: false}} } http_client_rtt_seconds: { description: "The round-trip time (RTT) of HTTP requests." @@ -500,19 +500,19 @@ components: sources: internal_metrics: output: metrics: { description: "The duration spent handling an HTTP request." type: "histogram" default_namespace: "vector" - tags: _component_tags & {method: _method, path: _path, status: _status} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, method: {description: "The HTTP method of the request.", required: false}, path: {description: "The path that produced the error.", required: true}, status: {description: "The HTTP status code of the request.", required: false}} } http_server_requests_received_total: { description: "The total number of HTTP requests received." type: "counter" default_namespace: "vector" - tags: _component_tags & {method: _method, path: _path} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, method: {description: "The HTTP method of the request.", required: false}, path: {description: "The path that produced the error.", required: true}} } http_server_responses_sent_total: { description: "The total number of HTTP responses sent." type: "counter" default_namespace: "vector" - tags: _component_tags & {method: _method, path: _path, status: _status} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, method: {description: "The HTTP method of the request.", required: false}, path: {description: "The path that produced the error.", required: true}, status: {description: "The HTTP status code of the request.", required: false}} } internal_metrics_cardinality: { description: "The total number of metrics emitted from the internal metrics registry." @@ -566,7 +566,7 @@ components: sources: internal_metrics: output: metrics: { description: "The Kafka consumer lag." type: "gauge" default_namespace: "vector" - tags: _component_tags & {topic_id: {description: "The Kafka topic id.", required: true}, partition_id: {description: "The Kafka partition id.", required: true}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, topic_id: {description: "The Kafka topic id.", required: true}, partition_id: {description: "The Kafka partition id.", required: true}} } kafka_produced_messages_bytes_total: { description: "Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers." @@ -620,7 +620,7 @@ components: sources: internal_metrics: output: metrics: { description: "The total memory currently being used by the Lua runtime." type: "gauge" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } memory_enrichment_table_byte_size: { description: "The total size in bytes of all objects stored in the in-memory enrichment table." @@ -686,7 +686,7 @@ components: sources: internal_metrics: output: metrics: { description: "The number of current open connections to Vector." type: "gauge" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } open_files: { description: "The total number of open files." @@ -704,13 +704,13 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of times the Vector instance has quit." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } reloaded_total: { description: "The total number of times the Vector instance has been reloaded." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } rewritten_timestamp_events_total: { description: "The total number of events with rewrapped timestamps." @@ -722,19 +722,19 @@ components: sources: internal_metrics: output: metrics: { description: "The time taken to process an S3 object that failed, in seconds." type: "histogram" default_namespace: "vector" - tags: _component_tags & {bucket: {description: "The name of the S3 bucket.", required: true}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, bucket: {description: "The name of the S3 bucket.", required: true}} } s3_object_processing_succeeded_duration_seconds: { description: "The time taken to process an S3 object that succeeded, in seconds." type: "histogram" default_namespace: "vector" - tags: _component_tags & {bucket: {description: "The name of the S3 bucket.", required: true}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, bucket: {description: "The name of the S3 bucket.", required: true}} } source_buffer_max_byte_size: { description: "The maximum number of bytes the source buffer can hold." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} deprecated: true deprecated_message: "This metric has been deprecated in favor of `source_buffer_max_size_bytes`." } @@ -742,7 +742,7 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum number of events the source buffer can hold." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} deprecated: true deprecated_message: "This metric has been deprecated in favor of `source_buffer_max_size_events`." } @@ -750,31 +750,31 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum number of bytes the source buffer can hold." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } source_buffer_max_size_events: { description: "The maximum number of events the source buffer can hold." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } source_buffer_utilization: { description: "The utilization of the source buffer." type: "histogram" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } source_buffer_utilization_level: { description: "The current utilization level of the source buffer." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } source_buffer_utilization_mean: { description: "The mean utilization of the source buffer, smoothed with an EWMA." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } source_lag_time_seconds: { description: "The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds." @@ -834,7 +834,7 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of times an S3 record in an SQS message was ignored." type: "counter" default_namespace: "vector" - tags: _component_tags & {ignore_type: {description: "The reason for ignoring the S3 record", required: true, enum: {invalid_event_kind: "The kind of invalid event."}}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, ignore_type: {description: "The reason for ignoring the S3 record", required: true, enum: {invalid_event_kind: "The kind of invalid event."}}} } stale_events_flushed_total: { description: "The number of stale events that Vector has flushed." @@ -846,13 +846,13 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of times the Vector instance has been started." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } stopped_total: { description: "The total number of times the Vector instance has been stopped." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } tag_cardinality_tracked_keys: { description: "The number of tag keys currently being tracked by the tag cardinality limit transform." @@ -870,13 +870,13 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`." type: "counter" default_namespace: "vector" - tags: _component_tags & {metric_name: {description: "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", required: false}, tag_key: {description: "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", required: false}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, metric_name: {description: "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", required: false}, tag_key: {description: "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", required: false}} } transform_buffer_max_byte_size: { description: "The maximum number of bytes the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} deprecated: true deprecated_message: "This metric has been deprecated in favor of `transform_buffer_max_size_bytes`." } @@ -884,7 +884,7 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum number of events the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} deprecated: true deprecated_message: "This metric has been deprecated in favor of `transform_buffer_max_size_events`." } @@ -892,43 +892,43 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum number of bytes the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } transform_buffer_max_size_events: { description: "The maximum number of events the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } transform_buffer_utilization: { description: "The utilization of the buffer that feeds into a transform." type: "histogram" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } transform_buffer_utilization_level: { description: "The current utilization level of the buffer that feeds into a transform." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } transform_buffer_utilization_mean: { description: "The mean utilization of the buffer that feeds into a transform, smoothed with an EWMA." type: "gauge" default_namespace: "vector" - tags: _component_tags & {output: _output} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} } uptime_seconds: { description: "The number of seconds the Vector instance has been running." type: "gauge" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } utf8_convert_errors_total: { description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." type: "counter" default_namespace: "vector" - tags: _component_tags & {mode: {description: "The connection mode used by the component.", required: true, enum: {udp: "User Datagram Protocol"}}} + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, mode: {description: "The connection mode used by the component.", required: true, enum: {udp: "User Datagram Protocol"}}} } utilization: { description: "The current utilization of this component, expressed as a value from 0 to 1." @@ -958,30 +958,30 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of times the Windows service has been installed." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } windows_service_restart_total: { description: "The total number of times the Windows service has been restarted." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } windows_service_start_total: { description: "The total number of times the Windows service has been started." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } windows_service_stop_total: { description: "The total number of times the Windows service has been stopped." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } windows_service_uninstall_total: { description: "The total number of times the Windows service has been uninstalled." type: "counter" default_namespace: "vector" - tags: _internal_metrics_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } } From 20eed32143136143ba3214a1e798909ce2377c35 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 13:36:18 -0400 Subject: [PATCH 22/43] fix(docs): restore collect features and how_it_works section accidentally dropped from internal_metrics.cue --- .../components/sources/internal_metrics.cue | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index 7fa9479afd896..442283de023a3 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -19,6 +19,10 @@ components: sources: internal_metrics: { features: { acknowledgements: false + collect: { + checkpoint: enabled: false + from: service: services.vector + } multiline: enabled: false } @@ -34,5 +38,18 @@ components: sources: internal_metrics: { configuration: generated.components.sources.internal_metrics.configuration - how_it_works: {} + how_it_works: { + unique_series: { + title: "Sending metrics from multiple Vector instances" + body: """ + When sending `internal_metrics` from multiple Vector instances + to the same destination, you will typically want to tag the + metrics with a tag that is unique to the Vector instance sending + the metrics to avoid the metric series conflicting. The + `tags.host_key` option can be used for this, but you can also + use a subsequent `remap` transform to add a different unique + tag from the environment. + """ + } + } } From ddf1b1caa1bb93ad70835195dc70ae656499b997 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 13:43:50 -0400 Subject: [PATCH 23/43] chore(metrics): inline all single-use tag field helpers --- .../src/internal_event/metric_tags.rs | 304 ++++++------------ 1 file changed, 106 insertions(+), 198 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index ce2f01726b9d3..27af50c0f8b6c 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -13,103 +13,15 @@ impl serde::Serialize for TagSet { } } -// ─── Individual tag field definitions ───────────────────────────────────────── - -fn field_pid() -> Value { - json!({"description": "The process ID of the Vector instance.", "required": false}) -} -fn field_host() -> Value { - json!({"description": "The hostname of the system Vector is running on.", "required": false}) -} -fn field_component_kind() -> Value { - json!({"description": "The Vector component kind.", "required": true, "enum": { - "sink": "Vector sink components", - "source": "Vector source components", - "transform": "Vector transform components" - }}) -} -fn field_component_id() -> Value { - json!({"description": "The Vector component ID.", "required": true}) -} -fn field_component_type() -> Value { - json!({"description": "The Vector component type.", "required": true}) -} -fn field_output() -> Value { - json!({"description": "The specific output of the component.", "required": false}) -} -fn field_mode_transport() -> Value { - json!({"description": "The connection mode used by the component.", "required": false, "enum": { - "udp": "User Datagram Protocol", - "tcp": "Transmission Control Protocol", - "unix": "Unix domain socket" - }}) -} -fn field_grpc_method() -> Value { - json!({"description": "The name of the method called on the gRPC service.", "required": true}) -} -fn field_grpc_service() -> Value { - json!({"description": "The gRPC service name.", "required": true}) -} -fn field_grpc_status() -> Value { - json!({"description": "The human-readable gRPC status code.", "required": true}) -} -fn field_http_status() -> Value { - json!({"description": "The HTTP status code of the request.", "required": false}) -} -fn field_http_method() -> Value { - json!({"description": "The HTTP method of the request.", "required": false}) -} -fn field_http_path() -> Value { - json!({"description": "The path that produced the error.", "required": true}) -} -fn field_file() -> Value { - json!({"description": "The file that produced the error.", "required": false}) -} -fn field_reason() -> Value { - json!({"description": "The type of the error", "required": true, "enum": { - "out_of_order": "The event was out of order.", - "oversized": "The event was too large." - }}) -} -fn field_error_type() -> Value { - json!({"description": "The type of the error", "required": true, "enum": { - "acknowledgements_failed": "The acknowledgement operation failed.", - "delete_failed": "The file deletion failed.", - "encode_failed": "The encode operation failed.", - "field_missing": "The event field was missing.", - "glob_failed": "The glob pattern match operation failed.", - "http_error": "The HTTP request resulted in an error code.", - "invalid_metric": "The metric was invalid.", - "kafka_offset_update": "The consumer offset update failed.", - "kafka_read": "The message from Kafka was invalid.", - "mapping_failed": "The mapping failed.", - "match_failed": "The match operation failed.", - "out_of_order": "The event was out of order.", - "parse_failed": "The parsing operation failed.", - "read_failed": "The file read operation failed.", - "render_error": "The rendering operation failed.", - "stream_closed": "The downstream was closed, forwarding the event(s) failed.", - "type_conversion_failed": "The type conversion operating failed.", - "type_field_does_not_exist": "The type field does not exist.", - "type_ip_address_parse_error": "The IP address did not parse.", - "unlabeled_event": "The event was not labeled.", - "value_invalid": "The value was invalid.", - "watch_failed": "The file watch operation failed.", - "write_failed": "The file write operation failed." - }}) -} -fn field_stage() -> Value { - json!({"description": "The stage within the component at which the error occurred.", "required": true, "enum": { - "receiving": "While receiving data.", - "processing": "While processing data within the component.", - "sending": "While sending data." - }}) -} - // ─── Base tag groups ─────────────────────────────────────────────────────────── fn internal_metrics_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| json!({"pid": field_pid(), "host": field_host()})); + static V: LazyLock = LazyLock::new(|| { + json!({ + "pid": {"description": "The process ID of the Vector instance.", "required": false}, + "host": {"description": "The hostname of the system Vector is running on.", "required": false} + }) + }); &V } @@ -117,9 +29,20 @@ fn component_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = internal_metrics_tags_value().clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("component_kind".to_owned(), field_component_kind()); - obj.insert("component_id".to_owned(), field_component_id()); - obj.insert("component_type".to_owned(), field_component_type()); + obj.insert( + "component_kind".to_owned(), + json!({ + "description": "The Vector component kind.", + "required": true, + "enum": { + "sink": "Vector sink components", + "source": "Vector source components", + "transform": "Vector transform components" + } + }), + ); + obj.insert("component_id".to_owned(), json!({"description": "The Vector component ID.", "required": true})); + obj.insert("component_type".to_owned(), json!({"description": "The Vector component type.", "required": true})); tags }); &V @@ -135,9 +58,8 @@ fn empty_tags_value() -> &'static Value { fn component_tags_output_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); - tags.as_object_mut() - .unwrap() - .insert("output".to_owned(), field_output()); + tags.as_object_mut().unwrap() + .insert("output".to_owned(), json!({"description": "The specific output of the component.", "required": false})); tags }); &V @@ -147,8 +69,8 @@ fn component_tags_grpc_method_service_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("grpc_method".to_owned(), field_grpc_method()); - obj.insert("grpc_service".to_owned(), field_grpc_service()); + obj.insert("grpc_method".to_owned(), json!({"description": "The name of the method called on the gRPC service.", "required": true})); + obj.insert("grpc_service".to_owned(), json!({"description": "The gRPC service name.", "required": true})); tags }); &V @@ -157,9 +79,8 @@ fn component_tags_grpc_method_service_value() -> &'static Value { fn component_tags_grpc_all_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_grpc_method_service_value().clone(); - tags.as_object_mut() - .unwrap() - .insert("grpc_status".to_owned(), field_grpc_status()); + tags.as_object_mut().unwrap() + .insert("grpc_status".to_owned(), json!({"description": "The human-readable gRPC status code.", "required": true})); tags }); &V @@ -168,9 +89,8 @@ fn component_tags_grpc_all_value() -> &'static Value { fn component_tags_http_method_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); - tags.as_object_mut() - .unwrap() - .insert("method".to_owned(), field_http_method()); + tags.as_object_mut().unwrap() + .insert("method".to_owned(), json!({"description": "The HTTP method of the request.", "required": false})); tags }); &V @@ -179,9 +99,8 @@ fn component_tags_http_method_value() -> &'static Value { fn component_tags_http_status_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); - tags.as_object_mut() - .unwrap() - .insert("status".to_owned(), field_http_status()); + tags.as_object_mut().unwrap() + .insert("status".to_owned(), json!({"description": "The HTTP status code of the request.", "required": false})); tags }); &V @@ -190,9 +109,8 @@ fn component_tags_http_status_value() -> &'static Value { fn component_tags_http_method_path_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_http_method_value().clone(); - tags.as_object_mut() - .unwrap() - .insert("path".to_owned(), field_http_path()); + tags.as_object_mut().unwrap() + .insert("path".to_owned(), json!({"description": "The path that produced the error.", "required": true})); tags }); &V @@ -201,9 +119,8 @@ fn component_tags_http_method_path_value() -> &'static Value { fn component_tags_http_all_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_http_method_path_value().clone(); - tags.as_object_mut() - .unwrap() - .insert("status".to_owned(), field_http_status()); + tags.as_object_mut().unwrap() + .insert("status".to_owned(), json!({"description": "The HTTP status code of the request.", "required": false})); tags }); &V @@ -213,8 +130,36 @@ fn component_tags_error_type_stage_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("error_type".to_owned(), field_error_type()); - obj.insert("stage".to_owned(), field_stage()); + obj.insert("error_type".to_owned(), json!({"description": "The type of the error", "required": true, "enum": { + "acknowledgements_failed": "The acknowledgement operation failed.", + "delete_failed": "The file deletion failed.", + "encode_failed": "The encode operation failed.", + "field_missing": "The event field was missing.", + "glob_failed": "The glob pattern match operation failed.", + "http_error": "The HTTP request resulted in an error code.", + "invalid_metric": "The metric was invalid.", + "kafka_offset_update": "The consumer offset update failed.", + "kafka_read": "The message from Kafka was invalid.", + "mapping_failed": "The mapping failed.", + "match_failed": "The match operation failed.", + "out_of_order": "The event was out of order.", + "parse_failed": "The parsing operation failed.", + "read_failed": "The file read operation failed.", + "render_error": "The rendering operation failed.", + "stream_closed": "The downstream was closed, forwarding the event(s) failed.", + "type_conversion_failed": "The type conversion operating failed.", + "type_field_does_not_exist": "The type field does not exist.", + "type_ip_address_parse_error": "The IP address did not parse.", + "unlabeled_event": "The event was not labeled.", + "value_invalid": "The value was invalid.", + "watch_failed": "The file watch operation failed.", + "write_failed": "The file write operation failed." + }})); + obj.insert("stage".to_owned(), json!({"description": "The stage within the component at which the error occurred.", "required": true, "enum": { + "receiving": "While receiving data.", + "processing": "While processing data within the component.", + "sending": "While sending data." + }})); tags }); &V @@ -225,9 +170,8 @@ fn component_tags_error_type_stage_value() -> &'static Value { fn internal_metrics_tags_file_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = internal_metrics_tags_value().clone(); - tags.as_object_mut() - .unwrap() - .insert("file".to_owned(), field_file()); + tags.as_object_mut().unwrap() + .insert("file".to_owned(), json!({"description": "The file that produced the error.", "required": false})); tags }); &V @@ -236,9 +180,11 @@ fn internal_metrics_tags_file_value() -> &'static Value { fn internal_metrics_tags_reason_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = internal_metrics_tags_value().clone(); - tags.as_object_mut() - .unwrap() - .insert("reason".to_owned(), field_reason()); + tags.as_object_mut().unwrap() + .insert("reason".to_owned(), json!({"description": "The type of the error", "required": true, "enum": { + "out_of_order": "The event was out of order.", + "oversized": "The event was too large." + }})); tags }); &V @@ -250,23 +196,17 @@ fn component_received_events_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert( - "file".to_owned(), - json!({"description": "The file from which the data originated.", "required": false}), - ); - obj.insert("uri".to_owned(), - json!({"description": "The sanitized URI from which the data originated.", "required": false})); - obj.insert("container_name".to_owned(), - json!({"description": "The name of the container from which the data originated.", "required": false})); - obj.insert("pod_name".to_owned(), - json!({"description": "The name of the pod from which the data originated.", "required": false})); - obj.insert( - "peer_addr".to_owned(), - json!({"description": "The IP from which the data originated.", "required": false}), - ); - obj.insert("peer_path".to_owned(), - json!({"description": "The pathname from which the data originated.", "required": false})); - obj.insert("mode".to_owned(), field_mode_transport()); + obj.insert("file".to_owned(), json!({"description": "The file from which the data originated.", "required": false})); + obj.insert("uri".to_owned(), json!({"description": "The sanitized URI from which the data originated.", "required": false})); + obj.insert("container_name".to_owned(), json!({"description": "The name of the container from which the data originated.", "required": false})); + obj.insert("pod_name".to_owned(), json!({"description": "The name of the pod from which the data originated.", "required": false})); + obj.insert("peer_addr".to_owned(), json!({"description": "The IP from which the data originated.", "required": false})); + obj.insert("peer_path".to_owned(), json!({"description": "The pathname from which the data originated.", "required": false})); + obj.insert("mode".to_owned(), json!({"description": "The connection mode used by the component.", "required": false, "enum": { + "udp": "User Datagram Protocol", + "tcp": "Transmission Control Protocol", + "unix": "Unix domain socket" + }})); tags }); &V @@ -289,14 +229,8 @@ fn component_sent_bytes_total_tags_value() -> &'static Value { let mut tags = component_tags_value().clone(); let obj = tags.as_object_mut().unwrap(); obj.insert("endpoint".to_owned(), json!({"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false})); - obj.insert( - "file".to_owned(), - json!({"description": "The absolute path of the destination file.", "required": false}), - ); - obj.insert( - "protocol".to_owned(), - json!({"description": "The protocol used to send the bytes.", "required": true}), - ); + obj.insert("file".to_owned(), json!({"description": "The absolute path of the destination file.", "required": false})); + obj.insert("protocol".to_owned(), json!({"description": "The protocol used to send the bytes.", "required": true})); obj.insert("region".to_owned(), json!({"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false})); tags }); @@ -306,10 +240,8 @@ fn component_sent_bytes_total_tags_value() -> &'static Value { fn s3_object_processing_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap().insert( - "bucket".to_owned(), - json!({"description": "The name of the S3 bucket.", "required": true}), - ); + tags.as_object_mut().unwrap() + .insert("bucket".to_owned(), json!({"description": "The name of the S3 bucket.", "required": true})); tags }); &V @@ -319,14 +251,8 @@ fn kafka_consumer_lag_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert( - "topic_id".to_owned(), - json!({"description": "The Kafka topic id.", "required": true}), - ); - obj.insert( - "partition_id".to_owned(), - json!({"description": "The Kafka partition id.", "required": true}), - ); + obj.insert("topic_id".to_owned(), json!({"description": "The Kafka topic id.", "required": true})); + obj.insert("partition_id".to_owned(), json!({"description": "The Kafka partition id.", "required": true})); tags }); &V @@ -346,14 +272,11 @@ fn tag_value_limit_exceeded_total_tags_value() -> &'static Value { fn sqs_s3_ignored_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap().insert( - "ignore_type".to_owned(), - json!({ - "description": "The reason for ignoring the S3 record", - "required": true, - "enum": {"invalid_event_kind": "The kind of invalid event."} - }), - ); + tags.as_object_mut().unwrap().insert("ignore_type".to_owned(), json!({ + "description": "The reason for ignoring the S3 record", + "required": true, + "enum": {"invalid_event_kind": "The kind of invalid event."} + })); tags }); &V @@ -363,18 +286,9 @@ fn build_info_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = internal_metrics_tags_value().clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert( - "debug".to_owned(), - json!({"description": "Whether this is a debug build of Vector", "required": true}), - ); - obj.insert( - "version".to_owned(), - json!({"description": "Vector version.", "required": true}), - ); - obj.insert( - "rust_version".to_owned(), - json!({"description": "The Rust version from the package manifest.", "required": true}), - ); + obj.insert("debug".to_owned(), json!({"description": "Whether this is a debug build of Vector", "required": true})); + obj.insert("version".to_owned(), json!({"description": "Vector version.", "required": true})); + obj.insert("rust_version".to_owned(), json!({"description": "The Rust version from the package manifest.", "required": true})); obj.insert("arch".to_owned(), json!({"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true})); obj.insert("revision".to_owned(), json!({"description": "Revision identifer, related to versioned releases.", "required": true})); tags @@ -385,14 +299,11 @@ fn build_info_tags_value() -> &'static Value { fn connection_read_errors_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap().insert( - "mode".to_owned(), - json!({ - "description": "", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - }), - ); + tags.as_object_mut().unwrap().insert("mode".to_owned(), json!({ + "description": "", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + })); tags }); &V @@ -401,14 +312,11 @@ fn connection_read_errors_total_tags_value() -> &'static Value { fn utf8_convert_errors_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap().insert( - "mode".to_owned(), - json!({ - "description": "The connection mode used by the component.", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - }), - ); + tags.as_object_mut().unwrap().insert("mode".to_owned(), json!({ + "description": "The connection mode used by the component.", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + })); tags }); &V From 6fe26c1bdf08f6f375d1682d7410c93cd290d418 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 14:03:33 -0400 Subject: [PATCH 24/43] chore(metrics): accept &* exprs in macro, use &*LAZY for annotation serialization --- .../src/internal_event/metric_name.rs | 42 +-- .../src/internal_event/metric_tags.rs | 252 +++++++++++------- lib/vector-config-macros/src/ast/mod.rs | 10 + 3 files changed, 180 insertions(+), 124 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index 45b911d735fdc..17b810d2b8939 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -90,11 +90,11 @@ pub enum CounterName { AggregateFlushesTotal, /// The number of times the Vector API has been started. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] ApiStartedTotal, /// The total number of files checkpointed. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] CheckpointsTotal, /// The total number of errors identifying files via checksum. @@ -102,22 +102,22 @@ pub enum CounterName { ChecksumErrorsTotal, /// The total number of metrics collections completed for this component. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] CollectCompletedTotal, /// The total number of times a command has been executed. CommandExecutedTotal, /// The total number of times a connection has been established. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] ConnectionEstablishedTotal, /// The total number of errors sending data via the connection. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] ConnectionSendErrorsTotal, /// The total number of times the connection has been shut down. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] ConnectionShutdownTotal, /// The total number of container events processed. @@ -228,11 +228,11 @@ pub enum CounterName { ParseErrorsTotal, /// The total number of times the Vector instance has quit. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] QuitTotal, /// The total number of times the Vector instance has been reloaded. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] ReloadedTotal, /// The total number of events with rewrapped timestamps. @@ -257,11 +257,11 @@ pub enum CounterName { StaleEventsFlushedTotal, /// The total number of times the Vector instance has been started. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] StartedTotal, /// The total number of times the Vector instance has been stopped. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] StoppedTotal, /// The total number of events that contained a tag which exceeded the configured cardinality limit. @@ -281,23 +281,23 @@ pub enum CounterName { WebsocketMessagesSentTotal, /// The total number of times the Windows service has been installed. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] WindowsServiceInstallTotal, /// The total number of times the Windows service has been restarted. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] WindowsServiceRestartTotal, /// The total number of times the Windows service has been started. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] WindowsServiceStartTotal, /// The total number of times the Windows service has been stopped. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] WindowsServiceStopTotal, /// The total number of times the Windows service has been uninstalled. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] WindowsServiceUninstallTotal, /// The total number of failures to annotate Kubernetes events with namespace metadata. @@ -381,7 +381,7 @@ pub enum HistogramName { /// /// This includes both the time spent queued in the transform's input buffer and the time spent /// executing the transform itself. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] ComponentLatencySeconds, /// The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds. @@ -423,7 +423,7 @@ pub enum HistogramName { S3ObjectProcessingFailedDurationSeconds, /// The duration spent collecting metrics for this component. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] CollectDurationSeconds, /// The command execution duration in seconds. @@ -504,7 +504,7 @@ pub enum GaugeName { /// This includes both the time spent queued in the transform's input buffer and the time spent /// executing the transform itself. This value is smoothed over time using an exponentially /// weighted moving average (EWMA). - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] ComponentLatencyMeanSeconds, /// The maximum number of events the source buffer can hold. @@ -605,7 +605,7 @@ pub enum GaugeName { OpenFiles, /// The number of seconds the Vector instance has been running. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] UptimeSeconds, /// Pseudo-metric that provides build information for the Vector instance. @@ -623,11 +623,11 @@ pub enum GaugeName { KafkaConsumerLag, /// The total memory currently being used by the Lua runtime. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] LuaMemoryUsedBytes, /// The number of current open connections to Vector. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] OpenConnections, /// The number of currently active endpoints. diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index 27af50c0f8b6c..c43a3bb73cd36 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -15,51 +15,48 @@ impl serde::Serialize for TagSet { // ─── Base tag groups ─────────────────────────────────────────────────────────── -fn internal_metrics_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { +pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { + json!({ + "pid": {"description": "The process ID of the Vector instance.", "required": false}, + "host": {"description": "The hostname of the system Vector is running on.", "required": false} + }) +}); + +pub static COMPONENT_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = INTERNAL_METRICS_TAGS.clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert( + "component_kind".to_owned(), json!({ - "pid": {"description": "The process ID of the Vector instance.", "required": false}, - "host": {"description": "The hostname of the system Vector is running on.", "required": false} - }) - }); - &V -} - -fn component_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = internal_metrics_tags_value().clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert( - "component_kind".to_owned(), - json!({ - "description": "The Vector component kind.", - "required": true, - "enum": { - "sink": "Vector sink components", - "source": "Vector source components", - "transform": "Vector transform components" - } - }), - ); - obj.insert("component_id".to_owned(), json!({"description": "The Vector component ID.", "required": true})); - obj.insert("component_type".to_owned(), json!({"description": "The Vector component type.", "required": true})); - tags - }); - &V -} - -fn empty_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| json!({})); - &V -} + "description": "The Vector component kind.", + "required": true, + "enum": { + "sink": "Vector sink components", + "source": "Vector source components", + "transform": "Vector transform components" + } + }), + ); + obj.insert( + "component_id".to_owned(), + json!({"description": "The Vector component ID.", "required": true}), + ); + obj.insert( + "component_type".to_owned(), + json!({"description": "The Vector component type.", "required": true}), + ); + tags +}); // ─── Extensions of COMPONENT_TAGS ───────────────────────────────────────────── fn component_tags_output_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap() - .insert("output".to_owned(), json!({"description": "The specific output of the component.", "required": false})); + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert( + "output".to_owned(), + json!({"description": "The specific output of the component.", "required": false}), + ); tags }); &V @@ -67,10 +64,13 @@ fn component_tags_output_value() -> &'static Value { fn component_tags_grpc_method_service_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); + let mut tags = COMPONENT_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); obj.insert("grpc_method".to_owned(), json!({"description": "The name of the method called on the gRPC service.", "required": true})); - obj.insert("grpc_service".to_owned(), json!({"description": "The gRPC service name.", "required": true})); + obj.insert( + "grpc_service".to_owned(), + json!({"description": "The gRPC service name.", "required": true}), + ); tags }); &V @@ -79,8 +79,10 @@ fn component_tags_grpc_method_service_value() -> &'static Value { fn component_tags_grpc_all_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_grpc_method_service_value().clone(); - tags.as_object_mut().unwrap() - .insert("grpc_status".to_owned(), json!({"description": "The human-readable gRPC status code.", "required": true})); + tags.as_object_mut().unwrap().insert( + "grpc_status".to_owned(), + json!({"description": "The human-readable gRPC status code.", "required": true}), + ); tags }); &V @@ -88,9 +90,11 @@ fn component_tags_grpc_all_value() -> &'static Value { fn component_tags_http_method_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap() - .insert("method".to_owned(), json!({"description": "The HTTP method of the request.", "required": false})); + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert( + "method".to_owned(), + json!({"description": "The HTTP method of the request.", "required": false}), + ); tags }); &V @@ -98,9 +102,11 @@ fn component_tags_http_method_value() -> &'static Value { fn component_tags_http_status_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap() - .insert("status".to_owned(), json!({"description": "The HTTP status code of the request.", "required": false})); + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert( + "status".to_owned(), + json!({"description": "The HTTP status code of the request.", "required": false}), + ); tags }); &V @@ -109,8 +115,10 @@ fn component_tags_http_status_value() -> &'static Value { fn component_tags_http_method_path_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_http_method_value().clone(); - tags.as_object_mut().unwrap() - .insert("path".to_owned(), json!({"description": "The path that produced the error.", "required": true})); + tags.as_object_mut().unwrap().insert( + "path".to_owned(), + json!({"description": "The path that produced the error.", "required": true}), + ); tags }); &V @@ -119,8 +127,10 @@ fn component_tags_http_method_path_value() -> &'static Value { fn component_tags_http_all_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { let mut tags = component_tags_http_method_path_value().clone(); - tags.as_object_mut().unwrap() - .insert("status".to_owned(), json!({"description": "The HTTP status code of the request.", "required": false})); + tags.as_object_mut().unwrap().insert( + "status".to_owned(), + json!({"description": "The HTTP status code of the request.", "required": false}), + ); tags }); &V @@ -128,7 +138,7 @@ fn component_tags_http_all_value() -> &'static Value { fn component_tags_error_type_stage_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); + let mut tags = COMPONENT_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); obj.insert("error_type".to_owned(), json!({"description": "The type of the error", "required": true, "enum": { "acknowledgements_failed": "The acknowledgement operation failed.", @@ -169,9 +179,11 @@ fn component_tags_error_type_stage_value() -> &'static Value { fn internal_metrics_tags_file_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = internal_metrics_tags_value().clone(); - tags.as_object_mut().unwrap() - .insert("file".to_owned(), json!({"description": "The file that produced the error.", "required": false})); + let mut tags = INTERNAL_METRICS_TAGS.clone(); + tags.as_object_mut().unwrap().insert( + "file".to_owned(), + json!({"description": "The file that produced the error.", "required": false}), + ); tags }); &V @@ -179,12 +191,14 @@ fn internal_metrics_tags_file_value() -> &'static Value { fn internal_metrics_tags_reason_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = internal_metrics_tags_value().clone(); - tags.as_object_mut().unwrap() - .insert("reason".to_owned(), json!({"description": "The type of the error", "required": true, "enum": { + let mut tags = INTERNAL_METRICS_TAGS.clone(); + tags.as_object_mut().unwrap().insert( + "reason".to_owned(), + json!({"description": "The type of the error", "required": true, "enum": { "out_of_order": "The event was out of order.", "oversized": "The event was too large." - }})); + }}), + ); tags }); &V @@ -194,13 +208,19 @@ fn internal_metrics_tags_reason_value() -> &'static Value { fn component_received_events_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); + let mut tags = COMPONENT_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("file".to_owned(), json!({"description": "The file from which the data originated.", "required": false})); + obj.insert( + "file".to_owned(), + json!({"description": "The file from which the data originated.", "required": false}), + ); obj.insert("uri".to_owned(), json!({"description": "The sanitized URI from which the data originated.", "required": false})); obj.insert("container_name".to_owned(), json!({"description": "The name of the container from which the data originated.", "required": false})); obj.insert("pod_name".to_owned(), json!({"description": "The name of the pod from which the data originated.", "required": false})); - obj.insert("peer_addr".to_owned(), json!({"description": "The IP from which the data originated.", "required": false})); + obj.insert( + "peer_addr".to_owned(), + json!({"description": "The IP from which the data originated.", "required": false}), + ); obj.insert("peer_path".to_owned(), json!({"description": "The pathname from which the data originated.", "required": false})); obj.insert("mode".to_owned(), json!({"description": "The connection mode used by the component.", "required": false, "enum": { "udp": "User Datagram Protocol", @@ -214,7 +234,7 @@ fn component_received_events_total_tags_value() -> &'static Value { fn component_discarded_events_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); + let mut tags = COMPONENT_TAGS.clone(); tags.as_object_mut().unwrap().insert("intentional".to_owned(), json!({ "description": "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", "required": true @@ -226,11 +246,17 @@ fn component_discarded_events_total_tags_value() -> &'static Value { fn component_sent_bytes_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); + let mut tags = COMPONENT_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); obj.insert("endpoint".to_owned(), json!({"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false})); - obj.insert("file".to_owned(), json!({"description": "The absolute path of the destination file.", "required": false})); - obj.insert("protocol".to_owned(), json!({"description": "The protocol used to send the bytes.", "required": true})); + obj.insert( + "file".to_owned(), + json!({"description": "The absolute path of the destination file.", "required": false}), + ); + obj.insert( + "protocol".to_owned(), + json!({"description": "The protocol used to send the bytes.", "required": true}), + ); obj.insert("region".to_owned(), json!({"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false})); tags }); @@ -239,9 +265,11 @@ fn component_sent_bytes_total_tags_value() -> &'static Value { fn s3_object_processing_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap() - .insert("bucket".to_owned(), json!({"description": "The name of the S3 bucket.", "required": true})); + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert( + "bucket".to_owned(), + json!({"description": "The name of the S3 bucket.", "required": true}), + ); tags }); &V @@ -249,10 +277,16 @@ fn s3_object_processing_tags_value() -> &'static Value { fn kafka_consumer_lag_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); + let mut tags = COMPONENT_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("topic_id".to_owned(), json!({"description": "The Kafka topic id.", "required": true})); - obj.insert("partition_id".to_owned(), json!({"description": "The Kafka partition id.", "required": true})); + obj.insert( + "topic_id".to_owned(), + json!({"description": "The Kafka topic id.", "required": true}), + ); + obj.insert( + "partition_id".to_owned(), + json!({"description": "The Kafka partition id.", "required": true}), + ); tags }); &V @@ -260,7 +294,7 @@ fn kafka_consumer_lag_tags_value() -> &'static Value { fn tag_value_limit_exceeded_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); + let mut tags = COMPONENT_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); obj.insert("metric_name".to_owned(), json!({"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); obj.insert("tag_key".to_owned(), json!({"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); @@ -271,12 +305,15 @@ fn tag_value_limit_exceeded_total_tags_value() -> &'static Value { fn sqs_s3_ignored_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap().insert("ignore_type".to_owned(), json!({ - "description": "The reason for ignoring the S3 record", - "required": true, - "enum": {"invalid_event_kind": "The kind of invalid event."} - })); + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert( + "ignore_type".to_owned(), + json!({ + "description": "The reason for ignoring the S3 record", + "required": true, + "enum": {"invalid_event_kind": "The kind of invalid event."} + }), + ); tags }); &V @@ -284,11 +321,20 @@ fn sqs_s3_ignored_total_tags_value() -> &'static Value { fn build_info_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = internal_metrics_tags_value().clone(); + let mut tags = INTERNAL_METRICS_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("debug".to_owned(), json!({"description": "Whether this is a debug build of Vector", "required": true})); - obj.insert("version".to_owned(), json!({"description": "Vector version.", "required": true})); - obj.insert("rust_version".to_owned(), json!({"description": "The Rust version from the package manifest.", "required": true})); + obj.insert( + "debug".to_owned(), + json!({"description": "Whether this is a debug build of Vector", "required": true}), + ); + obj.insert( + "version".to_owned(), + json!({"description": "Vector version.", "required": true}), + ); + obj.insert( + "rust_version".to_owned(), + json!({"description": "The Rust version from the package manifest.", "required": true}), + ); obj.insert("arch".to_owned(), json!({"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true})); obj.insert("revision".to_owned(), json!({"description": "Revision identifer, related to versioned releases.", "required": true})); tags @@ -298,12 +344,15 @@ fn build_info_tags_value() -> &'static Value { fn connection_read_errors_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap().insert("mode".to_owned(), json!({ - "description": "", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - })); + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert( + "mode".to_owned(), + json!({ + "description": "", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + }), + ); tags }); &V @@ -311,23 +360,20 @@ fn connection_read_errors_total_tags_value() -> &'static Value { fn utf8_convert_errors_total_tags_value() -> &'static Value { static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_value().clone(); - tags.as_object_mut().unwrap().insert("mode".to_owned(), json!({ - "description": "The connection mode used by the component.", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - })); + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert( + "mode".to_owned(), + json!({ + "description": "The connection mode used by the component.", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + }), + ); tags }); &V } -// ─── Public constants ────────────────────────────────────────────────────────── - -pub const INTERNAL_METRICS_TAGS: TagSet = TagSet(internal_metrics_tags_value); -pub const COMPONENT_TAGS: TagSet = TagSet(component_tags_value); -pub const EMPTY_TAGS: TagSet = TagSet(empty_tags_value); - /// Same tag set as `component_received_events_total` (inherited by byte-count metrics). pub const COMPONENT_RECEIVED_EVENTS_TAGS: TagSet = TagSet(component_received_events_total_tags_value); diff --git a/lib/vector-config-macros/src/ast/mod.rs b/lib/vector-config-macros/src/ast/mod.rs index 6b6a14f241fef..7528d192958c5 100644 --- a/lib/vector-config-macros/src/ast/mod.rs +++ b/lib/vector-config-macros/src/ast/mod.rs @@ -242,6 +242,16 @@ impl FromMeta for Metadata { key: path_to_string(&nv.path), value: mac.to_token_stream(), }), + // Accept reference/deref expressions such as `&*SOME_LAZY_LOCK` + // so callers can coerce `LazyLock` to `&T: Serialize`. + Expr::Reference(r) => Some(LazyCustomAttribute::KeyValue { + key: path_to_string(&nv.path), + value: r.to_token_stream(), + }), + Expr::Unary(u) => Some(LazyCustomAttribute::KeyValue { + key: path_to_string(&nv.path), + value: u.to_token_stream(), + }), expr => { errors .push(darling::Error::unexpected_expr_type(expr).with_span(nmeta)); From 539fbf6b62780662bea44ae1b76f33b84f671111 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 14:12:07 -0400 Subject: [PATCH 25/43] =?UTF-8?q?chore(metrics):=20remove=20TagSet=20?= =?UTF-8?q?=E2=80=94=20all=20tag=20sets=20are=20now=20pub=20static=20LazyL?= =?UTF-8?q?ock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/internal_event/metric_name.rs | 96 +-- .../src/internal_event/metric_tags.rs | 581 +++++++----------- 2 files changed, 268 insertions(+), 409 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index 17b810d2b8939..215f154b7b3c6 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -20,36 +20,36 @@ use super::metric_tags::{ pub enum CounterName { /// The number of events accepted by this component either from tagged /// origins like file and uri, or cumulatively from other origins. - #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = &*COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS))] ComponentReceivedEventsTotal, /// The number of event bytes accepted by this component either from /// tagged origins like file and uri, or cumulatively from other origins. - #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TAGS))] + #[configurable(metadata(docs::tags = &*COMPONENT_RECEIVED_EVENTS_TAGS))] ComponentReceivedEventBytesTotal, /// The number of raw bytes accepted by this component from source origins. - #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TAGS))] + #[configurable(metadata(docs::tags = &*COMPONENT_RECEIVED_EVENTS_TAGS))] ComponentReceivedBytesTotal, /// The total number of events emitted by this component. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] ComponentSentEventsTotal, /// The total number of event bytes emitted by this component. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] ComponentSentEventBytesTotal, /// The number of raw bytes sent by this component to destination sinks. - #[configurable(metadata(docs::tags = COMPONENT_SENT_BYTES_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = &*COMPONENT_SENT_BYTES_TOTAL_TAGS))] ComponentSentBytesTotal, /// The number of events dropped by this component. - #[configurable(metadata(docs::tags = COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = &*COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS))] ComponentDiscardedEventsTotal, /// The total number of errors encountered by this component. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_ERROR_TYPE_STAGE))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_ERROR_TYPE_STAGE))] ComponentErrorsTotal, /// The total number of events for which this source responded with a timeout error. @@ -98,7 +98,7 @@ pub enum CounterName { CheckpointsTotal, /// The total number of errors identifying files via checksum. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS_FILE))] ChecksumErrorsTotal, /// The total number of metrics collections completed for this component. @@ -148,50 +148,50 @@ pub enum CounterName { EncoderUnmappableReplacementWarningsTotal, /// The total number of events discarded by this component. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_REASON))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS_REASON))] EventsDiscardedTotal, /// The total number of files Vector has found to watch. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS_FILE))] FilesAddedTotal, /// The total number of files deleted. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS_FILE))] FilesDeletedTotal, /// The total number of times Vector has resumed watching a file. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS_FILE))] FilesResumedTotal, /// The total number of times Vector has stopped watching a file. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_FILE))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS_FILE))] FilesUnwatchedTotal, /// The total number of gRPC messages received. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_GRPC_METHOD_SERVICE))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_GRPC_METHOD_SERVICE))] GrpcServerMessagesReceivedTotal, /// The total number of gRPC messages sent. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_GRPC_ALL))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_GRPC_ALL))] GrpcServerMessagesSentTotal, /// The total number of HTTP client errors encountered. HttpClientErrorsTotal, /// The total number of sent HTTP requests, tagged with the request method. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_METHOD))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_HTTP_METHOD))] HttpClientRequestsSentTotal, /// The total number of HTTP requests, tagged with the response code. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_STATUS))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_HTTP_STATUS))] HttpClientResponsesTotal, /// The total number of HTTP requests received. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_METHOD_PATH))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_HTTP_METHOD_PATH))] HttpServerRequestsReceivedTotal, /// The total number of HTTP responses sent. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_ALL))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_HTTP_ALL))] HttpServerResponsesSentTotal, /// Total number of message bytes (including framing) received from Kafka brokers. @@ -268,7 +268,7 @@ pub enum CounterName { TagCardinalityUntrackedEventsTotal, /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. - #[configurable(metadata(docs::tags = TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = &*TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS))] TagValueLimitExceededTotal, /// The total number of times the value limit was reached. @@ -313,7 +313,7 @@ pub enum CounterName { K8sDockerFormatParseFailuresTotal, /// The total number of times an S3 record in an SQS message was ignored. - #[configurable(metadata(docs::tags = SQS_S3_IGNORED_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = &*SQS_S3_IGNORED_TOTAL_TAGS))] SqsS3EventRecordIgnoredTotal, /// The total number of bytes allocated by this component. @@ -341,7 +341,7 @@ pub enum CounterName { MemoryEnrichmentTableTtlExpirations, /// The total number of errors reading datagram. - #[configurable(metadata(docs::tags = CONNECTION_READ_ERRORS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = &*CONNECTION_READ_ERRORS_TOTAL_TAGS))] ConnectionReadErrorsTotal, /// The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`. @@ -349,11 +349,11 @@ pub enum CounterName { InternalMetricsCardinalityTotal, /// Number of configuration reload attempts that were rejected. - #[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS_REASON))] + #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS_REASON))] ConfigReloadRejected, /// The total number of errors converting bytes to a UTF-8 string in UDP mode. - #[configurable(metadata(docs::tags = UTF8_CONVERT_ERRORS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = &*UTF8_CONVERT_ERRORS_TOTAL_TAGS))] Utf8ConvertErrorsTotal, } @@ -367,11 +367,11 @@ pub enum HistogramName { /// /// Note that this is separate than sink-level batching. It is mostly useful for low level /// debugging performance issues in Vector due to small internal batches. - #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TAGS))] + #[configurable(metadata(docs::tags = &*COMPONENT_RECEIVED_EVENTS_TAGS))] ComponentReceivedEventsCount, /// The size in bytes of each event received by the source. - #[configurable(metadata(docs::tags = COMPONENT_RECEIVED_EVENTS_TAGS))] + #[configurable(metadata(docs::tags = &*COMPONENT_RECEIVED_EVENTS_TAGS))] ComponentReceivedBytes, /// The duration spent sending a payload to this buffer. @@ -415,11 +415,11 @@ pub enum HistogramName { AdaptiveConcurrencyReachedLimit, /// The time taken to process an S3 object that succeeded, in seconds. - #[configurable(metadata(docs::tags = S3_OBJECT_PROCESSING_TAGS))] + #[configurable(metadata(docs::tags = &*S3_OBJECT_PROCESSING_TAGS))] S3ObjectProcessingSucceededDurationSeconds, /// The time taken to process an S3 object that failed, in seconds. - #[configurable(metadata(docs::tags = S3_OBJECT_PROCESSING_TAGS))] + #[configurable(metadata(docs::tags = &*S3_OBJECT_PROCESSING_TAGS))] S3ObjectProcessingFailedDurationSeconds, /// The duration spent collecting metrics for this component. @@ -430,29 +430,29 @@ pub enum HistogramName { CommandExecutionDurationSeconds, /// The duration spent handling a gRPC request. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_GRPC_ALL))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_GRPC_ALL))] GrpcServerHandlerDurationSeconds, /// The duration spent handling an HTTP request. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_ALL))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_HTTP_ALL))] HttpServerHandlerDurationSeconds, /// The round-trip time (RTT) of HTTP requests. HttpClientRttSeconds, /// The round-trip time (RTT) of HTTP requests, tagged with the response code. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_HTTP_STATUS))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_HTTP_STATUS))] HttpClientResponseRttSeconds, /// The round-trip time (RTT) of HTTP requests that resulted in an error. HttpClientErrorRttSeconds, /// The utilization of the source buffer. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferUtilization, /// The utilization of the buffer that feeds into a transform. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferUtilization, } @@ -511,60 +511,60 @@ pub enum GaugeName { #[configurable( deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_events`." )] - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferMaxEventSize, /// The maximum number of bytes the source buffer can hold. #[configurable( deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_bytes`." )] - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferMaxByteSize, /// The maximum number of events the source buffer can hold. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferMaxSizeEvents, /// The maximum number of bytes the source buffer can hold. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferMaxSizeBytes, /// The current utilization level of the source buffer. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferUtilizationLevel, /// The mean utilization of the source buffer, smoothed with an EWMA. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferUtilizationMean, /// The maximum number of events the buffer that feeds into a transform can hold. #[configurable( deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_events`." )] - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferMaxEventSize, /// The maximum number of bytes the buffer that feeds into a transform can hold. #[configurable( deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_bytes`." )] - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferMaxByteSize, /// The maximum number of events the buffer that feeds into a transform can hold. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferMaxSizeEvents, /// The maximum number of bytes the buffer that feeds into a transform can hold. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferMaxSizeBytes, /// The current utilization level of the buffer that feeds into a transform. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferUtilizationLevel, /// The mean utilization of the buffer that feeds into a transform, smoothed with an EWMA. - #[configurable(metadata(docs::tags = COMPONENT_TAGS_OUTPUT))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferUtilizationMean, /// The maximum number of events in the buffer. @@ -609,7 +609,7 @@ pub enum GaugeName { UptimeSeconds, /// Pseudo-metric that provides build information for the Vector instance. - #[configurable(metadata(docs::tags = BUILD_INFO_TAGS))] + #[configurable(metadata(docs::tags = &*BUILD_INFO_TAGS))] BuildInfo, /// Current number of messages in producer queues. @@ -619,7 +619,7 @@ pub enum GaugeName { KafkaQueueMessagesBytes, /// The Kafka consumer lag. - #[configurable(metadata(docs::tags = KAFKA_CONSUMER_LAG_TAGS))] + #[configurable(metadata(docs::tags = &*KAFKA_CONSUMER_LAG_TAGS))] KafkaConsumerLag, /// The total memory currently being used by the Lua runtime. diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index c43a3bb73cd36..155c321cf6c49 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -1,18 +1,6 @@ use serde_json::{Value, json}; use std::sync::LazyLock; -/// A compile-time handle to a lazily-initialized JSON tag-set. -/// -/// Holds a function pointer so it can be used as a `const` in -/// `#[configurable(metadata(docs::tags = MY_TAGS))]` attributes. -pub struct TagSet(pub fn() -> &'static Value); - -impl serde::Serialize for TagSet { - fn serialize(&self, s: S) -> Result { - (self.0)().serialize(s) - } -} - // ─── Base tag groups ─────────────────────────────────────────────────────────── pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { @@ -25,381 +13,252 @@ pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { pub static COMPONENT_TAGS: LazyLock = LazyLock::new(|| { let mut tags = INTERNAL_METRICS_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert( - "component_kind".to_owned(), - json!({ - "description": "The Vector component kind.", - "required": true, - "enum": { - "sink": "Vector sink components", - "source": "Vector source components", - "transform": "Vector transform components" - } - }), - ); - obj.insert( - "component_id".to_owned(), - json!({"description": "The Vector component ID.", "required": true}), - ); - obj.insert( - "component_type".to_owned(), - json!({"description": "The Vector component type.", "required": true}), - ); + obj.insert("component_kind".to_owned(), json!({ + "description": "The Vector component kind.", + "required": true, + "enum": { + "sink": "Vector sink components", + "source": "Vector source components", + "transform": "Vector transform components" + } + })); + obj.insert("component_id".to_owned(), + json!({"description": "The Vector component ID.", "required": true})); + obj.insert("component_type".to_owned(), + json!({"description": "The Vector component type.", "required": true})); tags }); // ─── Extensions of COMPONENT_TAGS ───────────────────────────────────────────── -fn component_tags_output_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "output".to_owned(), - json!({"description": "The specific output of the component.", "required": false}), - ); - tags - }); - &V -} +pub static COMPONENT_TAGS_OUTPUT: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert("output".to_owned(), + json!({"description": "The specific output of the component.", "required": false})); + tags +}); -fn component_tags_grpc_method_service_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert("grpc_method".to_owned(), json!({"description": "The name of the method called on the gRPC service.", "required": true})); - obj.insert( - "grpc_service".to_owned(), - json!({"description": "The gRPC service name.", "required": true}), - ); - tags - }); - &V -} +pub static COMPONENT_TAGS_GRPC_METHOD_SERVICE: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("grpc_method".to_owned(), + json!({"description": "The name of the method called on the gRPC service.", "required": true})); + obj.insert("grpc_service".to_owned(), + json!({"description": "The gRPC service name.", "required": true})); + tags +}); -fn component_tags_grpc_all_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_grpc_method_service_value().clone(); - tags.as_object_mut().unwrap().insert( - "grpc_status".to_owned(), - json!({"description": "The human-readable gRPC status code.", "required": true}), - ); - tags - }); - &V -} +pub static COMPONENT_TAGS_GRPC_ALL: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS_GRPC_METHOD_SERVICE.clone(); + tags.as_object_mut().unwrap().insert("grpc_status".to_owned(), + json!({"description": "The human-readable gRPC status code.", "required": true})); + tags +}); -fn component_tags_http_method_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "method".to_owned(), - json!({"description": "The HTTP method of the request.", "required": false}), - ); - tags - }); - &V -} +pub static COMPONENT_TAGS_HTTP_METHOD: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert("method".to_owned(), + json!({"description": "The HTTP method of the request.", "required": false})); + tags +}); -fn component_tags_http_status_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "status".to_owned(), - json!({"description": "The HTTP status code of the request.", "required": false}), - ); - tags - }); - &V -} +pub static COMPONENT_TAGS_HTTP_STATUS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert("status".to_owned(), + json!({"description": "The HTTP status code of the request.", "required": false})); + tags +}); -fn component_tags_http_method_path_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_http_method_value().clone(); - tags.as_object_mut().unwrap().insert( - "path".to_owned(), - json!({"description": "The path that produced the error.", "required": true}), - ); - tags - }); - &V -} +pub static COMPONENT_TAGS_HTTP_METHOD_PATH: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS_HTTP_METHOD.clone(); + tags.as_object_mut().unwrap().insert("path".to_owned(), + json!({"description": "The path that produced the error.", "required": true})); + tags +}); -fn component_tags_http_all_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = component_tags_http_method_path_value().clone(); - tags.as_object_mut().unwrap().insert( - "status".to_owned(), - json!({"description": "The HTTP status code of the request.", "required": false}), - ); - tags - }); - &V -} +pub static COMPONENT_TAGS_HTTP_ALL: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS_HTTP_METHOD_PATH.clone(); + tags.as_object_mut().unwrap().insert("status".to_owned(), + json!({"description": "The HTTP status code of the request.", "required": false})); + tags +}); -fn component_tags_error_type_stage_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert("error_type".to_owned(), json!({"description": "The type of the error", "required": true, "enum": { - "acknowledgements_failed": "The acknowledgement operation failed.", - "delete_failed": "The file deletion failed.", - "encode_failed": "The encode operation failed.", - "field_missing": "The event field was missing.", - "glob_failed": "The glob pattern match operation failed.", - "http_error": "The HTTP request resulted in an error code.", - "invalid_metric": "The metric was invalid.", - "kafka_offset_update": "The consumer offset update failed.", - "kafka_read": "The message from Kafka was invalid.", - "mapping_failed": "The mapping failed.", - "match_failed": "The match operation failed.", - "out_of_order": "The event was out of order.", - "parse_failed": "The parsing operation failed.", - "read_failed": "The file read operation failed.", - "render_error": "The rendering operation failed.", - "stream_closed": "The downstream was closed, forwarding the event(s) failed.", - "type_conversion_failed": "The type conversion operating failed.", - "type_field_does_not_exist": "The type field does not exist.", - "type_ip_address_parse_error": "The IP address did not parse.", - "unlabeled_event": "The event was not labeled.", - "value_invalid": "The value was invalid.", - "watch_failed": "The file watch operation failed.", - "write_failed": "The file write operation failed." - }})); - obj.insert("stage".to_owned(), json!({"description": "The stage within the component at which the error occurred.", "required": true, "enum": { - "receiving": "While receiving data.", - "processing": "While processing data within the component.", - "sending": "While sending data." - }})); - tags - }); - &V -} +pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("error_type".to_owned(), json!({"description": "The type of the error", "required": true, "enum": { + "acknowledgements_failed": "The acknowledgement operation failed.", + "delete_failed": "The file deletion failed.", + "encode_failed": "The encode operation failed.", + "field_missing": "The event field was missing.", + "glob_failed": "The glob pattern match operation failed.", + "http_error": "The HTTP request resulted in an error code.", + "invalid_metric": "The metric was invalid.", + "kafka_offset_update": "The consumer offset update failed.", + "kafka_read": "The message from Kafka was invalid.", + "mapping_failed": "The mapping failed.", + "match_failed": "The match operation failed.", + "out_of_order": "The event was out of order.", + "parse_failed": "The parsing operation failed.", + "read_failed": "The file read operation failed.", + "render_error": "The rendering operation failed.", + "stream_closed": "The downstream was closed, forwarding the event(s) failed.", + "type_conversion_failed": "The type conversion operating failed.", + "type_field_does_not_exist": "The type field does not exist.", + "type_ip_address_parse_error": "The IP address did not parse.", + "unlabeled_event": "The event was not labeled.", + "value_invalid": "The value was invalid.", + "watch_failed": "The file watch operation failed.", + "write_failed": "The file write operation failed." + }})); + obj.insert("stage".to_owned(), json!({"description": "The stage within the component at which the error occurred.", "required": true, "enum": { + "receiving": "While receiving data.", + "processing": "While processing data within the component.", + "sending": "While sending data." + }})); + tags +}); // ─── Extensions of INTERNAL_METRICS_TAGS ────────────────────────────────────── -fn internal_metrics_tags_file_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = INTERNAL_METRICS_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "file".to_owned(), - json!({"description": "The file that produced the error.", "required": false}), - ); - tags - }); - &V -} +pub static INTERNAL_METRICS_TAGS_FILE: LazyLock = LazyLock::new(|| { + let mut tags = INTERNAL_METRICS_TAGS.clone(); + tags.as_object_mut().unwrap().insert("file".to_owned(), + json!({"description": "The file that produced the error.", "required": false})); + tags +}); -fn internal_metrics_tags_reason_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = INTERNAL_METRICS_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "reason".to_owned(), - json!({"description": "The type of the error", "required": true, "enum": { - "out_of_order": "The event was out of order.", - "oversized": "The event was too large." - }}), - ); - tags - }); - &V -} +pub static INTERNAL_METRICS_TAGS_REASON: LazyLock = LazyLock::new(|| { + let mut tags = INTERNAL_METRICS_TAGS.clone(); + tags.as_object_mut().unwrap().insert("reason".to_owned(), json!({ + "description": "The type of the error", + "required": true, + "enum": {"out_of_order": "The event was out of order.", "oversized": "The event was too large."} + })); + tags +}); // ─── Metric-specific tag sets ───────────────────────────────────────────────── -fn component_received_events_total_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert( - "file".to_owned(), - json!({"description": "The file from which the data originated.", "required": false}), - ); - obj.insert("uri".to_owned(), json!({"description": "The sanitized URI from which the data originated.", "required": false})); - obj.insert("container_name".to_owned(), json!({"description": "The name of the container from which the data originated.", "required": false})); - obj.insert("pod_name".to_owned(), json!({"description": "The name of the pod from which the data originated.", "required": false})); - obj.insert( - "peer_addr".to_owned(), - json!({"description": "The IP from which the data originated.", "required": false}), - ); - obj.insert("peer_path".to_owned(), json!({"description": "The pathname from which the data originated.", "required": false})); - obj.insert("mode".to_owned(), json!({"description": "The connection mode used by the component.", "required": false, "enum": { - "udp": "User Datagram Protocol", - "tcp": "Transmission Control Protocol", - "unix": "Unix domain socket" - }})); - tags - }); - &V -} - -fn component_discarded_events_total_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert("intentional".to_owned(), json!({ - "description": "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", - "required": true - })); - tags - }); - &V -} +pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("file".to_owned(), + json!({"description": "The file from which the data originated.", "required": false})); + obj.insert("uri".to_owned(), + json!({"description": "The sanitized URI from which the data originated.", "required": false})); + obj.insert("container_name".to_owned(), + json!({"description": "The name of the container from which the data originated.", "required": false})); + obj.insert("pod_name".to_owned(), + json!({"description": "The name of the pod from which the data originated.", "required": false})); + obj.insert("peer_addr".to_owned(), + json!({"description": "The IP from which the data originated.", "required": false})); + obj.insert("peer_path".to_owned(), + json!({"description": "The pathname from which the data originated.", "required": false})); + obj.insert("mode".to_owned(), json!({"description": "The connection mode used by the component.", "required": false, "enum": { + "udp": "User Datagram Protocol", + "tcp": "Transmission Control Protocol", + "unix": "Unix domain socket" + }})); + tags +}); -fn component_sent_bytes_total_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert("endpoint".to_owned(), json!({"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false})); - obj.insert( - "file".to_owned(), - json!({"description": "The absolute path of the destination file.", "required": false}), - ); - obj.insert( - "protocol".to_owned(), - json!({"description": "The protocol used to send the bytes.", "required": true}), - ); - obj.insert("region".to_owned(), json!({"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false})); - tags - }); - &V -} +/// Same tag set as `component_received_events_total` (inherited by byte-count metrics). +pub static COMPONENT_RECEIVED_EVENTS_TAGS: LazyLock = + LazyLock::new(|| COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS.clone()); -fn s3_object_processing_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "bucket".to_owned(), - json!({"description": "The name of the S3 bucket.", "required": true}), - ); - tags - }); - &V -} +pub static COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert("intentional".to_owned(), json!({ + "description": "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", + "required": true + })); + tags +}); -fn kafka_consumer_lag_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert( - "topic_id".to_owned(), - json!({"description": "The Kafka topic id.", "required": true}), - ); - obj.insert( - "partition_id".to_owned(), - json!({"description": "The Kafka partition id.", "required": true}), - ); - tags - }); - &V -} +pub static COMPONENT_SENT_BYTES_TOTAL_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("endpoint".to_owned(), + json!({"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false})); + obj.insert("file".to_owned(), + json!({"description": "The absolute path of the destination file.", "required": false})); + obj.insert("protocol".to_owned(), + json!({"description": "The protocol used to send the bytes.", "required": true})); + obj.insert("region".to_owned(), + json!({"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false})); + tags +}); -fn tag_value_limit_exceeded_total_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert("metric_name".to_owned(), json!({"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); - obj.insert("tag_key".to_owned(), json!({"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); - tags - }); - &V -} +pub static S3_OBJECT_PROCESSING_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert("bucket".to_owned(), + json!({"description": "The name of the S3 bucket.", "required": true})); + tags +}); -fn sqs_s3_ignored_total_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "ignore_type".to_owned(), - json!({ - "description": "The reason for ignoring the S3 record", - "required": true, - "enum": {"invalid_event_kind": "The kind of invalid event."} - }), - ); - tags - }); - &V -} +pub static KAFKA_CONSUMER_LAG_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("topic_id".to_owned(), + json!({"description": "The Kafka topic id.", "required": true})); + obj.insert("partition_id".to_owned(), + json!({"description": "The Kafka partition id.", "required": true})); + tags +}); -fn build_info_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = INTERNAL_METRICS_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert( - "debug".to_owned(), - json!({"description": "Whether this is a debug build of Vector", "required": true}), - ); - obj.insert( - "version".to_owned(), - json!({"description": "Vector version.", "required": true}), - ); - obj.insert( - "rust_version".to_owned(), - json!({"description": "The Rust version from the package manifest.", "required": true}), - ); - obj.insert("arch".to_owned(), json!({"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true})); - obj.insert("revision".to_owned(), json!({"description": "Revision identifer, related to versioned releases.", "required": true})); - tags - }); - &V -} +pub static TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("metric_name".to_owned(), + json!({"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); + obj.insert("tag_key".to_owned(), + json!({"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); + tags +}); -fn connection_read_errors_total_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "mode".to_owned(), - json!({ - "description": "", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - }), - ); - tags - }); - &V -} +pub static SQS_S3_IGNORED_TOTAL_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert("ignore_type".to_owned(), json!({ + "description": "The reason for ignoring the S3 record", + "required": true, + "enum": {"invalid_event_kind": "The kind of invalid event."} + })); + tags +}); -fn utf8_convert_errors_total_tags_value() -> &'static Value { - static V: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "mode".to_owned(), - json!({ - "description": "The connection mode used by the component.", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - }), - ); - tags - }); - &V -} +pub static BUILD_INFO_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = INTERNAL_METRICS_TAGS.clone(); + let obj = tags.as_object_mut().unwrap(); + obj.insert("debug".to_owned(), + json!({"description": "Whether this is a debug build of Vector", "required": true})); + obj.insert("version".to_owned(), + json!({"description": "Vector version.", "required": true})); + obj.insert("rust_version".to_owned(), + json!({"description": "The Rust version from the package manifest.", "required": true})); + obj.insert("arch".to_owned(), + json!({"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true})); + obj.insert("revision".to_owned(), + json!({"description": "Revision identifer, related to versioned releases.", "required": true})); + tags +}); -/// Same tag set as `component_received_events_total` (inherited by byte-count metrics). -pub const COMPONENT_RECEIVED_EVENTS_TAGS: TagSet = - TagSet(component_received_events_total_tags_value); +pub static CONNECTION_READ_ERRORS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert("mode".to_owned(), json!({ + "description": "", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + })); + tags +}); -pub const COMPONENT_TAGS_OUTPUT: TagSet = TagSet(component_tags_output_value); -pub const COMPONENT_TAGS_GRPC_METHOD_SERVICE: TagSet = - TagSet(component_tags_grpc_method_service_value); -pub const COMPONENT_TAGS_GRPC_ALL: TagSet = TagSet(component_tags_grpc_all_value); -pub const COMPONENT_TAGS_HTTP_METHOD: TagSet = TagSet(component_tags_http_method_value); -pub const COMPONENT_TAGS_HTTP_STATUS: TagSet = TagSet(component_tags_http_status_value); -pub const COMPONENT_TAGS_HTTP_METHOD_PATH: TagSet = TagSet(component_tags_http_method_path_value); -pub const COMPONENT_TAGS_HTTP_ALL: TagSet = TagSet(component_tags_http_all_value); -pub const COMPONENT_TAGS_ERROR_TYPE_STAGE: TagSet = TagSet(component_tags_error_type_stage_value); -pub const INTERNAL_METRICS_TAGS_FILE: TagSet = TagSet(internal_metrics_tags_file_value); -pub const INTERNAL_METRICS_TAGS_REASON: TagSet = TagSet(internal_metrics_tags_reason_value); -pub const COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: TagSet = - TagSet(component_received_events_total_tags_value); -pub const COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS: TagSet = - TagSet(component_discarded_events_total_tags_value); -pub const COMPONENT_SENT_BYTES_TOTAL_TAGS: TagSet = TagSet(component_sent_bytes_total_tags_value); -pub const S3_OBJECT_PROCESSING_TAGS: TagSet = TagSet(s3_object_processing_tags_value); -pub const KAFKA_CONSUMER_LAG_TAGS: TagSet = TagSet(kafka_consumer_lag_tags_value); -pub const TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS: TagSet = - TagSet(tag_value_limit_exceeded_total_tags_value); -pub const SQS_S3_IGNORED_TOTAL_TAGS: TagSet = TagSet(sqs_s3_ignored_total_tags_value); -pub const BUILD_INFO_TAGS: TagSet = TagSet(build_info_tags_value); -pub const CONNECTION_READ_ERRORS_TOTAL_TAGS: TagSet = - TagSet(connection_read_errors_total_tags_value); -pub const UTF8_CONVERT_ERRORS_TOTAL_TAGS: TagSet = TagSet(utf8_convert_errors_total_tags_value); +pub static UTF8_CONVERT_ERRORS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { + let mut tags = COMPONENT_TAGS.clone(); + tags.as_object_mut().unwrap().insert("mode".to_owned(), json!({ + "description": "The connection mode used by the component.", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + })); + tags +}); From 8e22c503eeef6dafc251cf9fea69a1f3b497e927 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 14:21:34 -0400 Subject: [PATCH 26/43] chore(metrics): add merge() helper, inline SQS_S3_IGNORED_TOTAL_TAGS as first example --- .../src/internal_event/metric_name.rs | 15 +- .../src/internal_event/metric_tags.rs | 201 +++++++++++------- lib/vector-config-macros/src/ast/mod.rs | 5 + 3 files changed, 146 insertions(+), 75 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index 215f154b7b3c6..a2bf65fd307e0 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -1,15 +1,16 @@ +use serde_json::json; use strum::{AsRefStr, Display, EnumIter}; use vector_config::configurable_component; use super::metric_tags::{ BUILD_INFO_TAGS, COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS, COMPONENT_RECEIVED_EVENTS_TAGS, - COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS, COMPONENT_SENT_BYTES_TOTAL_TAGS, + COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS, COMPONENT_SENT_BYTES_TOTAL_TAGS, COMPONENT_TAGS, COMPONENT_TAGS_ERROR_TYPE_STAGE, COMPONENT_TAGS_GRPC_ALL, COMPONENT_TAGS_GRPC_METHOD_SERVICE, COMPONENT_TAGS_HTTP_ALL, COMPONENT_TAGS_HTTP_METHOD, COMPONENT_TAGS_HTTP_METHOD_PATH, COMPONENT_TAGS_HTTP_STATUS, COMPONENT_TAGS_OUTPUT, CONNECTION_READ_ERRORS_TOTAL_TAGS, INTERNAL_METRICS_TAGS, INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, - KAFKA_CONSUMER_LAG_TAGS, S3_OBJECT_PROCESSING_TAGS, SQS_S3_IGNORED_TOTAL_TAGS, - TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS, UTF8_CONVERT_ERRORS_TOTAL_TAGS, + KAFKA_CONSUMER_LAG_TAGS, S3_OBJECT_PROCESSING_TAGS, TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS, + UTF8_CONVERT_ERRORS_TOTAL_TAGS, merge, }; /// Canonical list of all per-component internal counter metric names emitted by Vector. @@ -313,7 +314,13 @@ pub enum CounterName { K8sDockerFormatParseFailuresTotal, /// The total number of times an S3 record in an SQS message was ignored. - #[configurable(metadata(docs::tags = &*SQS_S3_IGNORED_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = merge(&*COMPONENT_TAGS, json!({ + "ignore_type": { + "description": "The reason for ignoring the S3 record", + "required": true, + "enum": {"invalid_event_kind": "The kind of invalid event."} + } + }))))] SqsS3EventRecordIgnoredTotal, /// The total number of bytes allocated by this component. diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index 155c321cf6c49..1aba228b69113 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -1,6 +1,24 @@ use serde_json::{Value, json}; use std::sync::LazyLock; +/// Clones `base` and inserts all fields from `extra`, returning the merged object. +/// +/// Intended for inline use in `#[configurable(metadata(docs::tags = ...))]` to avoid +/// naming single-use tag sets: +/// +/// ```ignore +/// #[configurable(metadata(docs::tags = merge(&*COMPONENT_TAGS, json!({ +/// "bucket": {"description": "The S3 bucket.", "required": true} +/// }))))] +/// ``` +pub fn merge(base: &Value, extra: Value) -> Value { + let mut result = base.clone(); + if let (Some(obj), Value::Object(extra_obj)) = (result.as_object_mut(), extra) { + obj.extend(extra_obj); + } + result +} + // ─── Base tag groups ─────────────────────────────────────────────────────────── pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { @@ -13,19 +31,26 @@ pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { pub static COMPONENT_TAGS: LazyLock = LazyLock::new(|| { let mut tags = INTERNAL_METRICS_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("component_kind".to_owned(), json!({ - "description": "The Vector component kind.", - "required": true, - "enum": { - "sink": "Vector sink components", - "source": "Vector source components", - "transform": "Vector transform components" - } - })); - obj.insert("component_id".to_owned(), - json!({"description": "The Vector component ID.", "required": true})); - obj.insert("component_type".to_owned(), - json!({"description": "The Vector component type.", "required": true})); + obj.insert( + "component_kind".to_owned(), + json!({ + "description": "The Vector component kind.", + "required": true, + "enum": { + "sink": "Vector sink components", + "source": "Vector source components", + "transform": "Vector transform components" + } + }), + ); + obj.insert( + "component_id".to_owned(), + json!({"description": "The Vector component ID.", "required": true}), + ); + obj.insert( + "component_type".to_owned(), + json!({"description": "The Vector component type.", "required": true}), + ); tags }); @@ -33,8 +58,10 @@ pub static COMPONENT_TAGS: LazyLock = LazyLock::new(|| { pub static COMPONENT_TAGS_OUTPUT: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert("output".to_owned(), - json!({"description": "The specific output of the component.", "required": false})); + tags.as_object_mut().unwrap().insert( + "output".to_owned(), + json!({"description": "The specific output of the component.", "required": false}), + ); tags }); @@ -43,43 +70,55 @@ pub static COMPONENT_TAGS_GRPC_METHOD_SERVICE: LazyLock = LazyLock::new(| let obj = tags.as_object_mut().unwrap(); obj.insert("grpc_method".to_owned(), json!({"description": "The name of the method called on the gRPC service.", "required": true})); - obj.insert("grpc_service".to_owned(), - json!({"description": "The gRPC service name.", "required": true})); + obj.insert( + "grpc_service".to_owned(), + json!({"description": "The gRPC service name.", "required": true}), + ); tags }); pub static COMPONENT_TAGS_GRPC_ALL: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS_GRPC_METHOD_SERVICE.clone(); - tags.as_object_mut().unwrap().insert("grpc_status".to_owned(), - json!({"description": "The human-readable gRPC status code.", "required": true})); + tags.as_object_mut().unwrap().insert( + "grpc_status".to_owned(), + json!({"description": "The human-readable gRPC status code.", "required": true}), + ); tags }); pub static COMPONENT_TAGS_HTTP_METHOD: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert("method".to_owned(), - json!({"description": "The HTTP method of the request.", "required": false})); + tags.as_object_mut().unwrap().insert( + "method".to_owned(), + json!({"description": "The HTTP method of the request.", "required": false}), + ); tags }); pub static COMPONENT_TAGS_HTTP_STATUS: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert("status".to_owned(), - json!({"description": "The HTTP status code of the request.", "required": false})); + tags.as_object_mut().unwrap().insert( + "status".to_owned(), + json!({"description": "The HTTP status code of the request.", "required": false}), + ); tags }); pub static COMPONENT_TAGS_HTTP_METHOD_PATH: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS_HTTP_METHOD.clone(); - tags.as_object_mut().unwrap().insert("path".to_owned(), - json!({"description": "The path that produced the error.", "required": true})); + tags.as_object_mut().unwrap().insert( + "path".to_owned(), + json!({"description": "The path that produced the error.", "required": true}), + ); tags }); pub static COMPONENT_TAGS_HTTP_ALL: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS_HTTP_METHOD_PATH.clone(); - tags.as_object_mut().unwrap().insert("status".to_owned(), - json!({"description": "The HTTP status code of the request.", "required": false})); + tags.as_object_mut().unwrap().insert( + "status".to_owned(), + json!({"description": "The HTTP status code of the request.", "required": false}), + ); tags }); @@ -123,8 +162,10 @@ pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock = LazyLock::new(|| { pub static INTERNAL_METRICS_TAGS_FILE: LazyLock = LazyLock::new(|| { let mut tags = INTERNAL_METRICS_TAGS.clone(); - tags.as_object_mut().unwrap().insert("file".to_owned(), - json!({"description": "The file that produced the error.", "required": false})); + tags.as_object_mut().unwrap().insert( + "file".to_owned(), + json!({"description": "The file that produced the error.", "required": false}), + ); tags }); @@ -143,18 +184,24 @@ pub static INTERNAL_METRICS_TAGS_REASON: LazyLock = LazyLock::new(|| { pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("file".to_owned(), - json!({"description": "The file from which the data originated.", "required": false})); + obj.insert( + "file".to_owned(), + json!({"description": "The file from which the data originated.", "required": false}), + ); obj.insert("uri".to_owned(), json!({"description": "The sanitized URI from which the data originated.", "required": false})); obj.insert("container_name".to_owned(), json!({"description": "The name of the container from which the data originated.", "required": false})); obj.insert("pod_name".to_owned(), json!({"description": "The name of the pod from which the data originated.", "required": false})); - obj.insert("peer_addr".to_owned(), - json!({"description": "The IP from which the data originated.", "required": false})); - obj.insert("peer_path".to_owned(), - json!({"description": "The pathname from which the data originated.", "required": false})); + obj.insert( + "peer_addr".to_owned(), + json!({"description": "The IP from which the data originated.", "required": false}), + ); + obj.insert( + "peer_path".to_owned(), + json!({"description": "The pathname from which the data originated.", "required": false}), + ); obj.insert("mode".to_owned(), json!({"description": "The connection mode used by the component.", "required": false, "enum": { "udp": "User Datagram Protocol", "tcp": "Transmission Control Protocol", @@ -181,10 +228,14 @@ pub static COMPONENT_SENT_BYTES_TOTAL_TAGS: LazyLock = LazyLock::new(|| { let obj = tags.as_object_mut().unwrap(); obj.insert("endpoint".to_owned(), json!({"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false})); - obj.insert("file".to_owned(), - json!({"description": "The absolute path of the destination file.", "required": false})); - obj.insert("protocol".to_owned(), - json!({"description": "The protocol used to send the bytes.", "required": true})); + obj.insert( + "file".to_owned(), + json!({"description": "The absolute path of the destination file.", "required": false}), + ); + obj.insert( + "protocol".to_owned(), + json!({"description": "The protocol used to send the bytes.", "required": true}), + ); obj.insert("region".to_owned(), json!({"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false})); tags @@ -192,18 +243,24 @@ pub static COMPONENT_SENT_BYTES_TOTAL_TAGS: LazyLock = LazyLock::new(|| { pub static S3_OBJECT_PROCESSING_TAGS: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert("bucket".to_owned(), - json!({"description": "The name of the S3 bucket.", "required": true})); + tags.as_object_mut().unwrap().insert( + "bucket".to_owned(), + json!({"description": "The name of the S3 bucket.", "required": true}), + ); tags }); pub static KAFKA_CONSUMER_LAG_TAGS: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("topic_id".to_owned(), - json!({"description": "The Kafka topic id.", "required": true})); - obj.insert("partition_id".to_owned(), - json!({"description": "The Kafka partition id.", "required": true})); + obj.insert( + "topic_id".to_owned(), + json!({"description": "The Kafka topic id.", "required": true}), + ); + obj.insert( + "partition_id".to_owned(), + json!({"description": "The Kafka partition id.", "required": true}), + ); tags }); @@ -217,25 +274,21 @@ pub static TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS: LazyLock = LazyLock::new( tags }); -pub static SQS_S3_IGNORED_TOTAL_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert("ignore_type".to_owned(), json!({ - "description": "The reason for ignoring the S3 record", - "required": true, - "enum": {"invalid_event_kind": "The kind of invalid event."} - })); - tags -}); - pub static BUILD_INFO_TAGS: LazyLock = LazyLock::new(|| { let mut tags = INTERNAL_METRICS_TAGS.clone(); let obj = tags.as_object_mut().unwrap(); - obj.insert("debug".to_owned(), - json!({"description": "Whether this is a debug build of Vector", "required": true})); - obj.insert("version".to_owned(), - json!({"description": "Vector version.", "required": true})); - obj.insert("rust_version".to_owned(), - json!({"description": "The Rust version from the package manifest.", "required": true})); + obj.insert( + "debug".to_owned(), + json!({"description": "Whether this is a debug build of Vector", "required": true}), + ); + obj.insert( + "version".to_owned(), + json!({"description": "Vector version.", "required": true}), + ); + obj.insert( + "rust_version".to_owned(), + json!({"description": "The Rust version from the package manifest.", "required": true}), + ); obj.insert("arch".to_owned(), json!({"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true})); obj.insert("revision".to_owned(), @@ -245,20 +298,26 @@ pub static BUILD_INFO_TAGS: LazyLock = LazyLock::new(|| { pub static CONNECTION_READ_ERRORS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert("mode".to_owned(), json!({ - "description": "", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - })); + tags.as_object_mut().unwrap().insert( + "mode".to_owned(), + json!({ + "description": "", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + }), + ); tags }); pub static UTF8_CONVERT_ERRORS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert("mode".to_owned(), json!({ - "description": "The connection mode used by the component.", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - })); + tags.as_object_mut().unwrap().insert( + "mode".to_owned(), + json!({ + "description": "The connection mode used by the component.", + "required": true, + "enum": {"udp": "User Datagram Protocol"} + }), + ); tags }); diff --git a/lib/vector-config-macros/src/ast/mod.rs b/lib/vector-config-macros/src/ast/mod.rs index 7528d192958c5..13ac0ea68c0ac 100644 --- a/lib/vector-config-macros/src/ast/mod.rs +++ b/lib/vector-config-macros/src/ast/mod.rs @@ -252,6 +252,11 @@ impl FromMeta for Metadata { key: path_to_string(&nv.path), value: u.to_token_stream(), }), + // Accept function calls such as `merge(&*BASE, json!({...}))`. + Expr::Call(c) => Some(LazyCustomAttribute::KeyValue { + key: path_to_string(&nv.path), + value: c.to_token_stream(), + }), expr => { errors .push(darling::Error::unexpected_expr_type(expr).with_span(nmeta)); From 49c6ae7b04311dd7d67707092f549665bcb97037 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 14:34:12 -0400 Subject: [PATCH 27/43] chore(metrics): inline 7 single-use tag statics with merge() --- .../src/internal_event/metric_name.rs | 48 ++++++--- .../src/internal_event/metric_tags.rs | 100 +----------------- 2 files changed, 35 insertions(+), 113 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index a2bf65fd307e0..8ea15155eec1c 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -3,14 +3,11 @@ use strum::{AsRefStr, Display, EnumIter}; use vector_config::configurable_component; use super::metric_tags::{ - BUILD_INFO_TAGS, COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS, COMPONENT_RECEIVED_EVENTS_TAGS, - COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS, COMPONENT_SENT_BYTES_TOTAL_TAGS, COMPONENT_TAGS, + COMPONENT_RECEIVED_EVENTS_TAGS, COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS, COMPONENT_TAGS, COMPONENT_TAGS_ERROR_TYPE_STAGE, COMPONENT_TAGS_GRPC_ALL, COMPONENT_TAGS_GRPC_METHOD_SERVICE, COMPONENT_TAGS_HTTP_ALL, COMPONENT_TAGS_HTTP_METHOD, COMPONENT_TAGS_HTTP_METHOD_PATH, - COMPONENT_TAGS_HTTP_STATUS, COMPONENT_TAGS_OUTPUT, CONNECTION_READ_ERRORS_TOTAL_TAGS, - INTERNAL_METRICS_TAGS, INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, - KAFKA_CONSUMER_LAG_TAGS, S3_OBJECT_PROCESSING_TAGS, TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS, - UTF8_CONVERT_ERRORS_TOTAL_TAGS, merge, + COMPONENT_TAGS_HTTP_STATUS, COMPONENT_TAGS_OUTPUT, INTERNAL_METRICS_TAGS, + INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, S3_OBJECT_PROCESSING_TAGS, merge, }; /// Canonical list of all per-component internal counter metric names emitted by Vector. @@ -42,11 +39,18 @@ pub enum CounterName { ComponentSentEventBytesTotal, /// The number of raw bytes sent by this component to destination sinks. - #[configurable(metadata(docs::tags = &*COMPONENT_SENT_BYTES_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + "endpoint": {"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false}, + "file": {"description": "The absolute path of the destination file.", "required": false}, + "protocol": {"description": "The protocol used to send the bytes.", "required": true}, + "region": {"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false} + }))))] ComponentSentBytesTotal, /// The number of events dropped by this component. - #[configurable(metadata(docs::tags = &*COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + "intentional": {"description": "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", "required": true} + }))))] ComponentDiscardedEventsTotal, /// The total number of errors encountered by this component. @@ -269,7 +273,10 @@ pub enum CounterName { TagCardinalityUntrackedEventsTotal, /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. - #[configurable(metadata(docs::tags = &*TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + "metric_name": {"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false}, + "tag_key": {"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false} + }))))] TagValueLimitExceededTotal, /// The total number of times the value limit was reached. @@ -314,7 +321,7 @@ pub enum CounterName { K8sDockerFormatParseFailuresTotal, /// The total number of times an S3 record in an SQS message was ignored. - #[configurable(metadata(docs::tags = merge(&*COMPONENT_TAGS, json!({ + #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ "ignore_type": { "description": "The reason for ignoring the S3 record", "required": true, @@ -348,7 +355,9 @@ pub enum CounterName { MemoryEnrichmentTableTtlExpirations, /// The total number of errors reading datagram. - #[configurable(metadata(docs::tags = &*CONNECTION_READ_ERRORS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + "mode": {"description": "", "required": true, "enum": {"udp": "User Datagram Protocol"}} + }))))] ConnectionReadErrorsTotal, /// The total number of metrics emitted from the internal metrics registry. This metric is deprecated in favor of `internal_metrics_cardinality`. @@ -360,7 +369,9 @@ pub enum CounterName { ConfigReloadRejected, /// The total number of errors converting bytes to a UTF-8 string in UDP mode. - #[configurable(metadata(docs::tags = &*UTF8_CONVERT_ERRORS_TOTAL_TAGS))] + #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + "mode": {"description": "The connection mode used by the component.", "required": true, "enum": {"udp": "User Datagram Protocol"}} + }))))] Utf8ConvertErrorsTotal, } @@ -616,7 +627,13 @@ pub enum GaugeName { UptimeSeconds, /// Pseudo-metric that provides build information for the Vector instance. - #[configurable(metadata(docs::tags = &*BUILD_INFO_TAGS))] + #[configurable(metadata(docs::tags = merge(&INTERNAL_METRICS_TAGS, json!({ + "debug": {"description": "Whether this is a debug build of Vector", "required": true}, + "version": {"description": "Vector version.", "required": true}, + "rust_version": {"description": "The Rust version from the package manifest.", "required": true}, + "arch": {"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true}, + "revision": {"description": "Revision identifer, related to versioned releases.", "required": true} + }))))] BuildInfo, /// Current number of messages in producer queues. @@ -626,7 +643,10 @@ pub enum GaugeName { KafkaQueueMessagesBytes, /// The Kafka consumer lag. - #[configurable(metadata(docs::tags = &*KAFKA_CONSUMER_LAG_TAGS))] + #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + "topic_id": {"description": "The Kafka topic id.", "required": true}, + "partition_id": {"description": "The Kafka partition id.", "required": true} + }))))] KafkaConsumerLag, /// The total memory currently being used by the Lua runtime. diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index 1aba228b69113..6ac532af88967 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -11,6 +11,7 @@ use std::sync::LazyLock; /// "bucket": {"description": "The S3 bucket.", "required": true} /// }))))] /// ``` +#[must_use] pub fn merge(base: &Value, extra: Value) -> Value { let mut result = base.clone(); if let (Some(obj), Value::Object(extra_obj)) = (result.as_object_mut(), extra) { @@ -214,33 +215,6 @@ pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new pub static COMPONENT_RECEIVED_EVENTS_TAGS: LazyLock = LazyLock::new(|| COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS.clone()); -pub static COMPONENT_DISCARDED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert("intentional".to_owned(), json!({ - "description": "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", - "required": true - })); - tags -}); - -pub static COMPONENT_SENT_BYTES_TOTAL_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert("endpoint".to_owned(), - json!({"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false})); - obj.insert( - "file".to_owned(), - json!({"description": "The absolute path of the destination file.", "required": false}), - ); - obj.insert( - "protocol".to_owned(), - json!({"description": "The protocol used to send the bytes.", "required": true}), - ); - obj.insert("region".to_owned(), - json!({"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false})); - tags -}); - pub static S3_OBJECT_PROCESSING_TAGS: LazyLock = LazyLock::new(|| { let mut tags = COMPONENT_TAGS.clone(); tags.as_object_mut().unwrap().insert( @@ -249,75 +223,3 @@ pub static S3_OBJECT_PROCESSING_TAGS: LazyLock = LazyLock::new(|| { ); tags }); - -pub static KAFKA_CONSUMER_LAG_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert( - "topic_id".to_owned(), - json!({"description": "The Kafka topic id.", "required": true}), - ); - obj.insert( - "partition_id".to_owned(), - json!({"description": "The Kafka partition id.", "required": true}), - ); - tags -}); - -pub static TAG_VALUE_LIMIT_EXCEEDED_TOTAL_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert("metric_name".to_owned(), - json!({"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); - obj.insert("tag_key".to_owned(), - json!({"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false})); - tags -}); - -pub static BUILD_INFO_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = INTERNAL_METRICS_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert( - "debug".to_owned(), - json!({"description": "Whether this is a debug build of Vector", "required": true}), - ); - obj.insert( - "version".to_owned(), - json!({"description": "Vector version.", "required": true}), - ); - obj.insert( - "rust_version".to_owned(), - json!({"description": "The Rust version from the package manifest.", "required": true}), - ); - obj.insert("arch".to_owned(), - json!({"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true})); - obj.insert("revision".to_owned(), - json!({"description": "Revision identifer, related to versioned releases.", "required": true})); - tags -}); - -pub static CONNECTION_READ_ERRORS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "mode".to_owned(), - json!({ - "description": "", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - }), - ); - tags -}); - -pub static UTF8_CONVERT_ERRORS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "mode".to_owned(), - json!({ - "description": "The connection mode used by the component.", - "required": true, - "enum": {"udp": "User Datagram Protocol"} - }), - ); - tags -}); From 1aa3bb0f2392cb0f4ce47d31e3755d994d211908 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 14:38:40 -0400 Subject: [PATCH 28/43] chore(metrics): introduce merge_lazy! macro, rewrite all statics with it --- .../src/internal_event/metric_tags.rs | 193 ++++++------------ 1 file changed, 63 insertions(+), 130 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index 6ac532af88967..f5eb634af0cc1 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -20,6 +20,19 @@ pub fn merge(base: &Value, extra: Value) -> Value { result } +/// Produces a `LazyLock` that merges extra fields into a cloned base tag set. +/// +/// ```ignore +/// pub static MY_TAGS: LazyLock = merge_lazy!(COMPONENT_TAGS, { +/// "bucket": {"description": "The S3 bucket.", "required": true} +/// }); +/// ``` +macro_rules! merge_lazy { + ($base:expr, { $($tt:tt)* }) => { + LazyLock::new(|| merge(&*$base, json!({ $($tt)* }))) + }; +} + // ─── Base tag groups ─────────────────────────────────────────────────────────── pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { @@ -29,104 +42,53 @@ pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { }) }); -pub static COMPONENT_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = INTERNAL_METRICS_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert( - "component_kind".to_owned(), - json!({ - "description": "The Vector component kind.", - "required": true, - "enum": { - "sink": "Vector sink components", - "source": "Vector source components", - "transform": "Vector transform components" - } - }), - ); - obj.insert( - "component_id".to_owned(), - json!({"description": "The Vector component ID.", "required": true}), - ); - obj.insert( - "component_type".to_owned(), - json!({"description": "The Vector component type.", "required": true}), - ); - tags +pub static COMPONENT_TAGS: LazyLock = merge_lazy!(INTERNAL_METRICS_TAGS, { + "component_kind": { + "description": "The Vector component kind.", + "required": true, + "enum": { + "sink": "Vector sink components", + "source": "Vector source components", + "transform": "Vector transform components" + } + }, + "component_id": {"description": "The Vector component ID.", "required": true}, + "component_type": {"description": "The Vector component type.", "required": true} }); // ─── Extensions of COMPONENT_TAGS ───────────────────────────────────────────── -pub static COMPONENT_TAGS_OUTPUT: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "output".to_owned(), - json!({"description": "The specific output of the component.", "required": false}), - ); - tags +pub static COMPONENT_TAGS_OUTPUT: LazyLock = merge_lazy!(COMPONENT_TAGS, { + "output": {"description": "The specific output of the component.", "required": false} }); -pub static COMPONENT_TAGS_GRPC_METHOD_SERVICE: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert("grpc_method".to_owned(), - json!({"description": "The name of the method called on the gRPC service.", "required": true})); - obj.insert( - "grpc_service".to_owned(), - json!({"description": "The gRPC service name.", "required": true}), - ); - tags +pub static COMPONENT_TAGS_GRPC_METHOD_SERVICE: LazyLock = merge_lazy!(COMPONENT_TAGS, { + "grpc_method": {"description": "The name of the method called on the gRPC service.", "required": true}, + "grpc_service": {"description": "The gRPC service name.", "required": true} }); -pub static COMPONENT_TAGS_GRPC_ALL: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS_GRPC_METHOD_SERVICE.clone(); - tags.as_object_mut().unwrap().insert( - "grpc_status".to_owned(), - json!({"description": "The human-readable gRPC status code.", "required": true}), - ); - tags +pub static COMPONENT_TAGS_GRPC_ALL: LazyLock = merge_lazy!(COMPONENT_TAGS_GRPC_METHOD_SERVICE, { + "grpc_status": {"description": "The human-readable gRPC status code.", "required": true} }); -pub static COMPONENT_TAGS_HTTP_METHOD: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "method".to_owned(), - json!({"description": "The HTTP method of the request.", "required": false}), - ); - tags +pub static COMPONENT_TAGS_HTTP_METHOD: LazyLock = merge_lazy!(COMPONENT_TAGS, { + "method": {"description": "The HTTP method of the request.", "required": false} }); -pub static COMPONENT_TAGS_HTTP_STATUS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "status".to_owned(), - json!({"description": "The HTTP status code of the request.", "required": false}), - ); - tags +pub static COMPONENT_TAGS_HTTP_STATUS: LazyLock = merge_lazy!(COMPONENT_TAGS, { + "status": {"description": "The HTTP status code of the request.", "required": false} }); -pub static COMPONENT_TAGS_HTTP_METHOD_PATH: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS_HTTP_METHOD.clone(); - tags.as_object_mut().unwrap().insert( - "path".to_owned(), - json!({"description": "The path that produced the error.", "required": true}), - ); - tags +pub static COMPONENT_TAGS_HTTP_METHOD_PATH: LazyLock = merge_lazy!(COMPONENT_TAGS_HTTP_METHOD, { + "path": {"description": "The path that produced the error.", "required": true} }); -pub static COMPONENT_TAGS_HTTP_ALL: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS_HTTP_METHOD_PATH.clone(); - tags.as_object_mut().unwrap().insert( - "status".to_owned(), - json!({"description": "The HTTP status code of the request.", "required": false}), - ); - tags +pub static COMPONENT_TAGS_HTTP_ALL: LazyLock = merge_lazy!(COMPONENT_TAGS_HTTP_METHOD_PATH, { + "status": {"description": "The HTTP status code of the request.", "required": false} }); -pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert("error_type".to_owned(), json!({"description": "The type of the error", "required": true, "enum": { +pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock = merge_lazy!(COMPONENT_TAGS, { + "error_type": {"description": "The type of the error", "required": true, "enum": { "acknowledgements_failed": "The acknowledgement operation failed.", "delete_failed": "The file deletion failed.", "encode_failed": "The encode operation failed.", @@ -150,76 +112,47 @@ pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock = LazyLock::new(|| { "value_invalid": "The value was invalid.", "watch_failed": "The file watch operation failed.", "write_failed": "The file write operation failed." - }})); - obj.insert("stage".to_owned(), json!({"description": "The stage within the component at which the error occurred.", "required": true, "enum": { + }}, + "stage": {"description": "The stage within the component at which the error occurred.", "required": true, "enum": { "receiving": "While receiving data.", "processing": "While processing data within the component.", "sending": "While sending data." - }})); - tags + }} }); // ─── Extensions of INTERNAL_METRICS_TAGS ────────────────────────────────────── -pub static INTERNAL_METRICS_TAGS_FILE: LazyLock = LazyLock::new(|| { - let mut tags = INTERNAL_METRICS_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "file".to_owned(), - json!({"description": "The file that produced the error.", "required": false}), - ); - tags +pub static INTERNAL_METRICS_TAGS_FILE: LazyLock = merge_lazy!(INTERNAL_METRICS_TAGS, { + "file": {"description": "The file that produced the error.", "required": false} }); -pub static INTERNAL_METRICS_TAGS_REASON: LazyLock = LazyLock::new(|| { - let mut tags = INTERNAL_METRICS_TAGS.clone(); - tags.as_object_mut().unwrap().insert("reason".to_owned(), json!({ - "description": "The type of the error", - "required": true, - "enum": {"out_of_order": "The event was out of order.", "oversized": "The event was too large."} - })); - tags +pub static INTERNAL_METRICS_TAGS_REASON: LazyLock = merge_lazy!(INTERNAL_METRICS_TAGS, { + "reason": {"description": "The type of the error", "required": true, "enum": { + "out_of_order": "The event was out of order.", + "oversized": "The event was too large." + }} }); // ─── Metric-specific tag sets ───────────────────────────────────────────────── -pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - let obj = tags.as_object_mut().unwrap(); - obj.insert( - "file".to_owned(), - json!({"description": "The file from which the data originated.", "required": false}), - ); - obj.insert("uri".to_owned(), - json!({"description": "The sanitized URI from which the data originated.", "required": false})); - obj.insert("container_name".to_owned(), - json!({"description": "The name of the container from which the data originated.", "required": false})); - obj.insert("pod_name".to_owned(), - json!({"description": "The name of the pod from which the data originated.", "required": false})); - obj.insert( - "peer_addr".to_owned(), - json!({"description": "The IP from which the data originated.", "required": false}), - ); - obj.insert( - "peer_path".to_owned(), - json!({"description": "The pathname from which the data originated.", "required": false}), - ); - obj.insert("mode".to_owned(), json!({"description": "The connection mode used by the component.", "required": false, "enum": { +pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = merge_lazy!(COMPONENT_TAGS, { + "file": {"description": "The file from which the data originated.", "required": false}, + "uri": {"description": "The sanitized URI from which the data originated.", "required": false}, + "container_name": {"description": "The name of the container from which the data originated.", "required": false}, + "pod_name": {"description": "The name of the pod from which the data originated.", "required": false}, + "peer_addr": {"description": "The IP from which the data originated.", "required": false}, + "peer_path": {"description": "The pathname from which the data originated.", "required": false}, + "mode": {"description": "The connection mode used by the component.", "required": false, "enum": { "udp": "User Datagram Protocol", "tcp": "Transmission Control Protocol", "unix": "Unix domain socket" - }})); - tags + }} }); /// Same tag set as `component_received_events_total` (inherited by byte-count metrics). pub static COMPONENT_RECEIVED_EVENTS_TAGS: LazyLock = LazyLock::new(|| COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS.clone()); -pub static S3_OBJECT_PROCESSING_TAGS: LazyLock = LazyLock::new(|| { - let mut tags = COMPONENT_TAGS.clone(); - tags.as_object_mut().unwrap().insert( - "bucket".to_owned(), - json!({"description": "The name of the S3 bucket.", "required": true}), - ); - tags +pub static S3_OBJECT_PROCESSING_TAGS: LazyLock = merge_lazy!(COMPONENT_TAGS, { + "bucket": {"description": "The name of the S3 bucket.", "required": true} }); From 4aec549cdc2f6cd5ebf8dbec2d3bc0b6bdb1c7fa Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 14:41:49 -0400 Subject: [PATCH 29/43] chore(metrics): replace merge_lazy! macro with merge_lazy() function --- .../src/internal_event/metric_name.rs | 19 +- .../src/internal_event/metric_tags.rs | 253 +++++++++++------- 2 files changed, 160 insertions(+), 112 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index 8ea15155eec1c..428d5a013e45b 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -7,7 +7,8 @@ use super::metric_tags::{ COMPONENT_TAGS_ERROR_TYPE_STAGE, COMPONENT_TAGS_GRPC_ALL, COMPONENT_TAGS_GRPC_METHOD_SERVICE, COMPONENT_TAGS_HTTP_ALL, COMPONENT_TAGS_HTTP_METHOD, COMPONENT_TAGS_HTTP_METHOD_PATH, COMPONENT_TAGS_HTTP_STATUS, COMPONENT_TAGS_OUTPUT, INTERNAL_METRICS_TAGS, - INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, S3_OBJECT_PROCESSING_TAGS, merge, + INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, S3_OBJECT_PROCESSING_TAGS, + merge_lazy, }; /// Canonical list of all per-component internal counter metric names emitted by Vector. @@ -39,7 +40,7 @@ pub enum CounterName { ComponentSentEventBytesTotal, /// The number of raw bytes sent by this component to destination sinks. - #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "endpoint": {"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false}, "file": {"description": "The absolute path of the destination file.", "required": false}, "protocol": {"description": "The protocol used to send the bytes.", "required": true}, @@ -48,7 +49,7 @@ pub enum CounterName { ComponentSentBytesTotal, /// The number of events dropped by this component. - #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "intentional": {"description": "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", "required": true} }))))] ComponentDiscardedEventsTotal, @@ -273,7 +274,7 @@ pub enum CounterName { TagCardinalityUntrackedEventsTotal, /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. - #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "metric_name": {"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false}, "tag_key": {"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false} }))))] @@ -321,7 +322,7 @@ pub enum CounterName { K8sDockerFormatParseFailuresTotal, /// The total number of times an S3 record in an SQS message was ignored. - #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "ignore_type": { "description": "The reason for ignoring the S3 record", "required": true, @@ -355,7 +356,7 @@ pub enum CounterName { MemoryEnrichmentTableTtlExpirations, /// The total number of errors reading datagram. - #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "mode": {"description": "", "required": true, "enum": {"udp": "User Datagram Protocol"}} }))))] ConnectionReadErrorsTotal, @@ -369,7 +370,7 @@ pub enum CounterName { ConfigReloadRejected, /// The total number of errors converting bytes to a UTF-8 string in UDP mode. - #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "mode": {"description": "The connection mode used by the component.", "required": true, "enum": {"udp": "User Datagram Protocol"}} }))))] Utf8ConvertErrorsTotal, @@ -627,7 +628,7 @@ pub enum GaugeName { UptimeSeconds, /// Pseudo-metric that provides build information for the Vector instance. - #[configurable(metadata(docs::tags = merge(&INTERNAL_METRICS_TAGS, json!({ + #[configurable(metadata(docs::tags = merge_lazy(&INTERNAL_METRICS_TAGS, json!({ "debug": {"description": "Whether this is a debug build of Vector", "required": true}, "version": {"description": "Vector version.", "required": true}, "rust_version": {"description": "The Rust version from the package manifest.", "required": true}, @@ -643,7 +644,7 @@ pub enum GaugeName { KafkaQueueMessagesBytes, /// The Kafka consumer lag. - #[configurable(metadata(docs::tags = merge(&COMPONENT_TAGS, json!({ + #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "topic_id": {"description": "The Kafka topic id.", "required": true}, "partition_id": {"description": "The Kafka partition id.", "required": true} }))))] diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index f5eb634af0cc1..3986064d85ca7 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -1,38 +1,20 @@ use serde_json::{Value, json}; use std::sync::LazyLock; -/// Clones `base` and inserts all fields from `extra`, returning the merged object. +/// Clones `base` (a `LazyLock`) and inserts all fields from `extra`. /// -/// Intended for inline use in `#[configurable(metadata(docs::tags = ...))]` to avoid -/// naming single-use tag sets: -/// -/// ```ignore -/// #[configurable(metadata(docs::tags = merge(&*COMPONENT_TAGS, json!({ -/// "bucket": {"description": "The S3 bucket.", "required": true} -/// }))))] -/// ``` +/// Intended for: +/// - static initializers: `LazyLock::new(|| merge_lazy(&BASE, json!({...})))` +/// - inline annotations: `merge_lazy(&COMPONENT_TAGS, json!({...}))` #[must_use] -pub fn merge(base: &Value, extra: Value) -> Value { - let mut result = base.clone(); +pub fn merge_lazy(base: &LazyLock, extra: Value) -> Value { + let mut result = (**base).clone(); if let (Some(obj), Value::Object(extra_obj)) = (result.as_object_mut(), extra) { obj.extend(extra_obj); } result } -/// Produces a `LazyLock` that merges extra fields into a cloned base tag set. -/// -/// ```ignore -/// pub static MY_TAGS: LazyLock = merge_lazy!(COMPONENT_TAGS, { -/// "bucket": {"description": "The S3 bucket.", "required": true} -/// }); -/// ``` -macro_rules! merge_lazy { - ($base:expr, { $($tt:tt)* }) => { - LazyLock::new(|| merge(&*$base, json!({ $($tt)* }))) - }; -} - // ─── Base tag groups ─────────────────────────────────────────────────────────── pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { @@ -42,117 +24,182 @@ pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { }) }); -pub static COMPONENT_TAGS: LazyLock = merge_lazy!(INTERNAL_METRICS_TAGS, { - "component_kind": { - "description": "The Vector component kind.", - "required": true, - "enum": { - "sink": "Vector sink components", - "source": "Vector source components", - "transform": "Vector transform components" - } - }, - "component_id": {"description": "The Vector component ID.", "required": true}, - "component_type": {"description": "The Vector component type.", "required": true} +pub static COMPONENT_TAGS: LazyLock = LazyLock::new(|| { + merge_lazy( + &INTERNAL_METRICS_TAGS, + json!({ + "component_kind": { + "description": "The Vector component kind.", + "required": true, + "enum": { + "sink": "Vector sink components", + "source": "Vector source components", + "transform": "Vector transform components" + } + }, + "component_id": {"description": "The Vector component ID.", "required": true}, + "component_type": {"description": "The Vector component type.", "required": true} + }), + ) }); // ─── Extensions of COMPONENT_TAGS ───────────────────────────────────────────── -pub static COMPONENT_TAGS_OUTPUT: LazyLock = merge_lazy!(COMPONENT_TAGS, { - "output": {"description": "The specific output of the component.", "required": false} +pub static COMPONENT_TAGS_OUTPUT: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS, + json!({ + "output": {"description": "The specific output of the component.", "required": false} + }), + ) }); -pub static COMPONENT_TAGS_GRPC_METHOD_SERVICE: LazyLock = merge_lazy!(COMPONENT_TAGS, { - "grpc_method": {"description": "The name of the method called on the gRPC service.", "required": true}, - "grpc_service": {"description": "The gRPC service name.", "required": true} +pub static COMPONENT_TAGS_GRPC_METHOD_SERVICE: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS, + json!({ + "grpc_method": {"description": "The name of the method called on the gRPC service.", "required": true}, + "grpc_service": {"description": "The gRPC service name.", "required": true} + }), + ) }); -pub static COMPONENT_TAGS_GRPC_ALL: LazyLock = merge_lazy!(COMPONENT_TAGS_GRPC_METHOD_SERVICE, { - "grpc_status": {"description": "The human-readable gRPC status code.", "required": true} +pub static COMPONENT_TAGS_GRPC_ALL: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS_GRPC_METHOD_SERVICE, + json!({ + "grpc_status": {"description": "The human-readable gRPC status code.", "required": true} + }), + ) }); -pub static COMPONENT_TAGS_HTTP_METHOD: LazyLock = merge_lazy!(COMPONENT_TAGS, { - "method": {"description": "The HTTP method of the request.", "required": false} +pub static COMPONENT_TAGS_HTTP_METHOD: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS, + json!({ + "method": {"description": "The HTTP method of the request.", "required": false} + }), + ) }); -pub static COMPONENT_TAGS_HTTP_STATUS: LazyLock = merge_lazy!(COMPONENT_TAGS, { - "status": {"description": "The HTTP status code of the request.", "required": false} +pub static COMPONENT_TAGS_HTTP_STATUS: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS, + json!({ + "status": {"description": "The HTTP status code of the request.", "required": false} + }), + ) }); -pub static COMPONENT_TAGS_HTTP_METHOD_PATH: LazyLock = merge_lazy!(COMPONENT_TAGS_HTTP_METHOD, { - "path": {"description": "The path that produced the error.", "required": true} +pub static COMPONENT_TAGS_HTTP_METHOD_PATH: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS_HTTP_METHOD, + json!({ + "path": {"description": "The path that produced the error.", "required": true} + }), + ) }); -pub static COMPONENT_TAGS_HTTP_ALL: LazyLock = merge_lazy!(COMPONENT_TAGS_HTTP_METHOD_PATH, { - "status": {"description": "The HTTP status code of the request.", "required": false} +pub static COMPONENT_TAGS_HTTP_ALL: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS_HTTP_METHOD_PATH, + json!({ + "status": {"description": "The HTTP status code of the request.", "required": false} + }), + ) }); -pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock = merge_lazy!(COMPONENT_TAGS, { - "error_type": {"description": "The type of the error", "required": true, "enum": { - "acknowledgements_failed": "The acknowledgement operation failed.", - "delete_failed": "The file deletion failed.", - "encode_failed": "The encode operation failed.", - "field_missing": "The event field was missing.", - "glob_failed": "The glob pattern match operation failed.", - "http_error": "The HTTP request resulted in an error code.", - "invalid_metric": "The metric was invalid.", - "kafka_offset_update": "The consumer offset update failed.", - "kafka_read": "The message from Kafka was invalid.", - "mapping_failed": "The mapping failed.", - "match_failed": "The match operation failed.", - "out_of_order": "The event was out of order.", - "parse_failed": "The parsing operation failed.", - "read_failed": "The file read operation failed.", - "render_error": "The rendering operation failed.", - "stream_closed": "The downstream was closed, forwarding the event(s) failed.", - "type_conversion_failed": "The type conversion operating failed.", - "type_field_does_not_exist": "The type field does not exist.", - "type_ip_address_parse_error": "The IP address did not parse.", - "unlabeled_event": "The event was not labeled.", - "value_invalid": "The value was invalid.", - "watch_failed": "The file watch operation failed.", - "write_failed": "The file write operation failed." - }}, - "stage": {"description": "The stage within the component at which the error occurred.", "required": true, "enum": { - "receiving": "While receiving data.", - "processing": "While processing data within the component.", - "sending": "While sending data." - }} +pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS, + json!({ + "error_type": {"description": "The type of the error", "required": true, "enum": { + "acknowledgements_failed": "The acknowledgement operation failed.", + "delete_failed": "The file deletion failed.", + "encode_failed": "The encode operation failed.", + "field_missing": "The event field was missing.", + "glob_failed": "The glob pattern match operation failed.", + "http_error": "The HTTP request resulted in an error code.", + "invalid_metric": "The metric was invalid.", + "kafka_offset_update": "The consumer offset update failed.", + "kafka_read": "The message from Kafka was invalid.", + "mapping_failed": "The mapping failed.", + "match_failed": "The match operation failed.", + "out_of_order": "The event was out of order.", + "parse_failed": "The parsing operation failed.", + "read_failed": "The file read operation failed.", + "render_error": "The rendering operation failed.", + "stream_closed": "The downstream was closed, forwarding the event(s) failed.", + "type_conversion_failed": "The type conversion operating failed.", + "type_field_does_not_exist": "The type field does not exist.", + "type_ip_address_parse_error": "The IP address did not parse.", + "unlabeled_event": "The event was not labeled.", + "value_invalid": "The value was invalid.", + "watch_failed": "The file watch operation failed.", + "write_failed": "The file write operation failed." + }}, + "stage": {"description": "The stage within the component at which the error occurred.", "required": true, "enum": { + "receiving": "While receiving data.", + "processing": "While processing data within the component.", + "sending": "While sending data." + }} + }), + ) }); // ─── Extensions of INTERNAL_METRICS_TAGS ────────────────────────────────────── -pub static INTERNAL_METRICS_TAGS_FILE: LazyLock = merge_lazy!(INTERNAL_METRICS_TAGS, { - "file": {"description": "The file that produced the error.", "required": false} +pub static INTERNAL_METRICS_TAGS_FILE: LazyLock = LazyLock::new(|| { + merge_lazy( + &INTERNAL_METRICS_TAGS, + json!({ + "file": {"description": "The file that produced the error.", "required": false} + }), + ) }); -pub static INTERNAL_METRICS_TAGS_REASON: LazyLock = merge_lazy!(INTERNAL_METRICS_TAGS, { - "reason": {"description": "The type of the error", "required": true, "enum": { - "out_of_order": "The event was out of order.", - "oversized": "The event was too large." - }} +pub static INTERNAL_METRICS_TAGS_REASON: LazyLock = LazyLock::new(|| { + merge_lazy( + &INTERNAL_METRICS_TAGS, + json!({ + "reason": {"description": "The type of the error", "required": true, "enum": { + "out_of_order": "The event was out of order.", + "oversized": "The event was too large." + }} + }), + ) }); // ─── Metric-specific tag sets ───────────────────────────────────────────────── -pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = merge_lazy!(COMPONENT_TAGS, { - "file": {"description": "The file from which the data originated.", "required": false}, - "uri": {"description": "The sanitized URI from which the data originated.", "required": false}, - "container_name": {"description": "The name of the container from which the data originated.", "required": false}, - "pod_name": {"description": "The name of the pod from which the data originated.", "required": false}, - "peer_addr": {"description": "The IP from which the data originated.", "required": false}, - "peer_path": {"description": "The pathname from which the data originated.", "required": false}, - "mode": {"description": "The connection mode used by the component.", "required": false, "enum": { - "udp": "User Datagram Protocol", - "tcp": "Transmission Control Protocol", - "unix": "Unix domain socket" - }} +pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS, + json!({ + "file": {"description": "The file from which the data originated.", "required": false}, + "uri": {"description": "The sanitized URI from which the data originated.", "required": false}, + "container_name": {"description": "The name of the container from which the data originated.", "required": false}, + "pod_name": {"description": "The name of the pod from which the data originated.", "required": false}, + "peer_addr": {"description": "The IP from which the data originated.", "required": false}, + "peer_path": {"description": "The pathname from which the data originated.", "required": false}, + "mode": {"description": "The connection mode used by the component.", "required": false, "enum": { + "udp": "User Datagram Protocol", + "tcp": "Transmission Control Protocol", + "unix": "Unix domain socket" + }} + }), + ) }); /// Same tag set as `component_received_events_total` (inherited by byte-count metrics). pub static COMPONENT_RECEIVED_EVENTS_TAGS: LazyLock = LazyLock::new(|| COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS.clone()); -pub static S3_OBJECT_PROCESSING_TAGS: LazyLock = merge_lazy!(COMPONENT_TAGS, { - "bucket": {"description": "The name of the S3 bucket.", "required": true} +pub static S3_OBJECT_PROCESSING_TAGS: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS, + json!({ + "bucket": {"description": "The name of the S3 bucket.", "required": true} + }), + ) }); From 205481e2eb33509fc1c87f87afffc824b9340214 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 14:49:55 -0400 Subject: [PATCH 30/43] Remove whitespace --- .../src/internal_event/metric_tags.rs | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index 3986064d85ca7..5a91f1052a554 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -5,7 +5,7 @@ use std::sync::LazyLock; /// /// Intended for: /// - static initializers: `LazyLock::new(|| merge_lazy(&BASE, json!({...})))` -/// - inline annotations: `merge_lazy(&COMPONENT_TAGS, json!({...}))` +/// - inline annotations: `merge_lazy(&COMPONENT_TAGS, json!({...}))` #[must_use] pub fn merge_lazy(base: &LazyLock, extra: Value) -> Value { let mut result = (**base).clone(); @@ -19,7 +19,7 @@ pub fn merge_lazy(base: &LazyLock, extra: Value) -> Value { pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { json!({ - "pid": {"description": "The process ID of the Vector instance.", "required": false}, + "pid": {"description": "The process ID of the Vector instance.", "required": false}, "host": {"description": "The hostname of the system Vector is running on.", "required": false} }) }); @@ -32,12 +32,12 @@ pub static COMPONENT_TAGS: LazyLock = LazyLock::new(|| { "description": "The Vector component kind.", "required": true, "enum": { - "sink": "Vector sink components", - "source": "Vector source components", + "sink": "Vector sink components", + "source": "Vector source components", "transform": "Vector transform components" } }, - "component_id": {"description": "The Vector component ID.", "required": true}, + "component_id": {"description": "The Vector component ID.", "required": true}, "component_type": {"description": "The Vector component type.", "required": true} }), ) @@ -58,7 +58,7 @@ pub static COMPONENT_TAGS_GRPC_METHOD_SERVICE: LazyLock = LazyLock::new(| merge_lazy( &COMPONENT_TAGS, json!({ - "grpc_method": {"description": "The name of the method called on the gRPC service.", "required": true}, + "grpc_method": {"description": "The name of the method called on the gRPC service.", "required": true}, "grpc_service": {"description": "The gRPC service name.", "required": true} }), ) @@ -114,34 +114,34 @@ pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock = LazyLock::new(|| { &COMPONENT_TAGS, json!({ "error_type": {"description": "The type of the error", "required": true, "enum": { - "acknowledgements_failed": "The acknowledgement operation failed.", - "delete_failed": "The file deletion failed.", - "encode_failed": "The encode operation failed.", - "field_missing": "The event field was missing.", - "glob_failed": "The glob pattern match operation failed.", - "http_error": "The HTTP request resulted in an error code.", - "invalid_metric": "The metric was invalid.", - "kafka_offset_update": "The consumer offset update failed.", - "kafka_read": "The message from Kafka was invalid.", - "mapping_failed": "The mapping failed.", - "match_failed": "The match operation failed.", - "out_of_order": "The event was out of order.", - "parse_failed": "The parsing operation failed.", - "read_failed": "The file read operation failed.", - "render_error": "The rendering operation failed.", - "stream_closed": "The downstream was closed, forwarding the event(s) failed.", - "type_conversion_failed": "The type conversion operating failed.", - "type_field_does_not_exist": "The type field does not exist.", + "acknowledgements_failed": "The acknowledgement operation failed.", + "delete_failed": "The file deletion failed.", + "encode_failed": "The encode operation failed.", + "field_missing": "The event field was missing.", + "glob_failed": "The glob pattern match operation failed.", + "http_error": "The HTTP request resulted in an error code.", + "invalid_metric": "The metric was invalid.", + "kafka_offset_update": "The consumer offset update failed.", + "kafka_read": "The message from Kafka was invalid.", + "mapping_failed": "The mapping failed.", + "match_failed": "The match operation failed.", + "out_of_order": "The event was out of order.", + "parse_failed": "The parsing operation failed.", + "read_failed": "The file read operation failed.", + "render_error": "The rendering operation failed.", + "stream_closed": "The downstream was closed, forwarding the event(s) failed.", + "type_conversion_failed": "The type conversion operating failed.", + "type_field_does_not_exist": "The type field does not exist.", "type_ip_address_parse_error": "The IP address did not parse.", - "unlabeled_event": "The event was not labeled.", - "value_invalid": "The value was invalid.", - "watch_failed": "The file watch operation failed.", - "write_failed": "The file write operation failed." + "unlabeled_event": "The event was not labeled.", + "value_invalid": "The value was invalid.", + "watch_failed": "The file watch operation failed.", + "write_failed": "The file write operation failed." }}, "stage": {"description": "The stage within the component at which the error occurred.", "required": true, "enum": { - "receiving": "While receiving data.", + "receiving": "While receiving data.", "processing": "While processing data within the component.", - "sending": "While sending data." + "sending": "While sending data." }} }), ) @@ -164,7 +164,7 @@ pub static INTERNAL_METRICS_TAGS_REASON: LazyLock = LazyLock::new(|| { json!({ "reason": {"description": "The type of the error", "required": true, "enum": { "out_of_order": "The event was out of order.", - "oversized": "The event was too large." + "oversized": "The event was too large." }} }), ) @@ -176,15 +176,15 @@ pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new merge_lazy( &COMPONENT_TAGS, json!({ - "file": {"description": "The file from which the data originated.", "required": false}, - "uri": {"description": "The sanitized URI from which the data originated.", "required": false}, + "file": {"description": "The file from which the data originated.", "required": false}, + "uri": {"description": "The sanitized URI from which the data originated.", "required": false}, "container_name": {"description": "The name of the container from which the data originated.", "required": false}, - "pod_name": {"description": "The name of the pod from which the data originated.", "required": false}, - "peer_addr": {"description": "The IP from which the data originated.", "required": false}, - "peer_path": {"description": "The pathname from which the data originated.", "required": false}, - "mode": {"description": "The connection mode used by the component.", "required": false, "enum": { - "udp": "User Datagram Protocol", - "tcp": "Transmission Control Protocol", + "pod_name": {"description": "The name of the pod from which the data originated.", "required": false}, + "peer_addr": {"description": "The IP from which the data originated.", "required": false}, + "peer_path": {"description": "The pathname from which the data originated.", "required": false}, + "mode": {"description": "The connection mode used by the component.", "required": false, "enum": { + "udp": "User Datagram Protocol", + "tcp": "Transmission Control Protocol", "unix": "Unix domain socket" }} }), From acdd77e4b6222b6945d73441129e36039e3d5400 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 14:58:03 -0400 Subject: [PATCH 31/43] Fix whitespace --- .../src/internal_event/metric_name.rs | 16 ++-- .../src/internal_event/metric_tags.rs | 94 +++++++++++-------- 2 files changed, 63 insertions(+), 47 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index 428d5a013e45b..d7e4d9200ce49 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -42,9 +42,9 @@ pub enum CounterName { /// The number of raw bytes sent by this component to destination sinks. #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "endpoint": {"description": "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", "required": false}, - "file": {"description": "The absolute path of the destination file.", "required": false}, + "file": {"description": "The absolute path of the destination file.", "required": false}, "protocol": {"description": "The protocol used to send the bytes.", "required": true}, - "region": {"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false} + "region": {"description": "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", "required": false} }))))] ComponentSentBytesTotal, @@ -276,7 +276,7 @@ pub enum CounterName { /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "metric_name": {"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false}, - "tag_key": {"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false} + "tag_key": {"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false} }))))] TagValueLimitExceededTotal, @@ -629,11 +629,11 @@ pub enum GaugeName { /// Pseudo-metric that provides build information for the Vector instance. #[configurable(metadata(docs::tags = merge_lazy(&INTERNAL_METRICS_TAGS, json!({ - "debug": {"description": "Whether this is a debug build of Vector", "required": true}, - "version": {"description": "Vector version.", "required": true}, + "debug": {"description": "Whether this is a debug build of Vector", "required": true}, + "version": {"description": "Vector version.", "required": true}, "rust_version": {"description": "The Rust version from the package manifest.", "required": true}, - "arch": {"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true}, - "revision": {"description": "Revision identifer, related to versioned releases.", "required": true} + "arch": {"description": "The target architecture being compiled for. (e.g. x86_64)", "required": true}, + "revision": {"description": "Revision identifer, related to versioned releases.", "required": true} }))))] BuildInfo, @@ -645,7 +645,7 @@ pub enum GaugeName { /// The Kafka consumer lag. #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ - "topic_id": {"description": "The Kafka topic id.", "required": true}, + "topic_id": {"description": "The Kafka topic id.", "required": true}, "partition_id": {"description": "The Kafka partition id.", "required": true} }))))] KafkaConsumerLag, diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index 5a91f1052a554..4b5e3a3255782 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -113,36 +113,44 @@ pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock = LazyLock::new(|| { merge_lazy( &COMPONENT_TAGS, json!({ - "error_type": {"description": "The type of the error", "required": true, "enum": { - "acknowledgements_failed": "The acknowledgement operation failed.", - "delete_failed": "The file deletion failed.", - "encode_failed": "The encode operation failed.", - "field_missing": "The event field was missing.", - "glob_failed": "The glob pattern match operation failed.", - "http_error": "The HTTP request resulted in an error code.", - "invalid_metric": "The metric was invalid.", - "kafka_offset_update": "The consumer offset update failed.", - "kafka_read": "The message from Kafka was invalid.", - "mapping_failed": "The mapping failed.", - "match_failed": "The match operation failed.", - "out_of_order": "The event was out of order.", - "parse_failed": "The parsing operation failed.", - "read_failed": "The file read operation failed.", - "render_error": "The rendering operation failed.", - "stream_closed": "The downstream was closed, forwarding the event(s) failed.", - "type_conversion_failed": "The type conversion operating failed.", - "type_field_does_not_exist": "The type field does not exist.", - "type_ip_address_parse_error": "The IP address did not parse.", - "unlabeled_event": "The event was not labeled.", - "value_invalid": "The value was invalid.", - "watch_failed": "The file watch operation failed.", - "write_failed": "The file write operation failed." - }}, - "stage": {"description": "The stage within the component at which the error occurred.", "required": true, "enum": { - "receiving": "While receiving data.", - "processing": "While processing data within the component.", - "sending": "While sending data." - }} + "error_type": { + "description": "The type of the error", + "required": true, + "enum": { + "acknowledgements_failed": "The acknowledgement operation failed.", + "delete_failed": "The file deletion failed.", + "encode_failed": "The encode operation failed.", + "field_missing": "The event field was missing.", + "glob_failed": "The glob pattern match operation failed.", + "http_error": "The HTTP request resulted in an error code.", + "invalid_metric": "The metric was invalid.", + "kafka_offset_update": "The consumer offset update failed.", + "kafka_read": "The message from Kafka was invalid.", + "mapping_failed": "The mapping failed.", + "match_failed": "The match operation failed.", + "out_of_order": "The event was out of order.", + "parse_failed": "The parsing operation failed.", + "read_failed": "The file read operation failed.", + "render_error": "The rendering operation failed.", + "stream_closed": "The downstream was closed, forwarding the event(s) failed.", + "type_conversion_failed": "The type conversion operating failed.", + "type_field_does_not_exist": "The type field does not exist.", + "type_ip_address_parse_error": "The IP address did not parse.", + "unlabeled_event": "The event was not labeled.", + "value_invalid": "The value was invalid.", + "watch_failed": "The file watch operation failed.", + "write_failed": "The file write operation failed." + } + }, + "stage": { + "description": "The stage within the component at which the error occurred.", + "required": true, + "enum": { + "receiving": "While receiving data.", + "processing": "While processing data within the component.", + "sending": "While sending data." + } + } }), ) }); @@ -162,10 +170,14 @@ pub static INTERNAL_METRICS_TAGS_REASON: LazyLock = LazyLock::new(|| { merge_lazy( &INTERNAL_METRICS_TAGS, json!({ - "reason": {"description": "The type of the error", "required": true, "enum": { - "out_of_order": "The event was out of order.", - "oversized": "The event was too large." - }} + "reason": { + "description": "The type of the error", + "required": true, + "enum": { + "out_of_order": "The event was out of order.", + "oversized": "The event was too large." + } + } }), ) }); @@ -182,11 +194,15 @@ pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new "pod_name": {"description": "The name of the pod from which the data originated.", "required": false}, "peer_addr": {"description": "The IP from which the data originated.", "required": false}, "peer_path": {"description": "The pathname from which the data originated.", "required": false}, - "mode": {"description": "The connection mode used by the component.", "required": false, "enum": { - "udp": "User Datagram Protocol", - "tcp": "Transmission Control Protocol", - "unix": "Unix domain socket" - }} + "mode": { + "description": "The connection mode used by the component.", + "required": false, + "enum": { + "udp": "User Datagram Protocol", + "tcp": "Transmission Control Protocol", + "unix": "Unix domain socket" + } + } }), ) }); From e5e870fb05a2047e806e51324e529887e4af613b Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 15:06:05 -0400 Subject: [PATCH 32/43] fix(vdev): warn and skip instead of falling back to stale _component_tags CUE reference --- vdev/src/commands/build/component_docs/runner.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index 4f3f0bf183f9b..bd5002d07f17b 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -467,7 +467,14 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { Some(Value::Object(obj)) => json_tags_to_cue(obj), // String values are plain CUE expressions emitted verbatim (e.g. "{}"). Some(Value::String(s)) => s.clone(), - _ => "_component_tags".to_owned(), + _ => { + warn!( + "metric variant '{}' has no docs::tags metadata — \ + add #[configurable(metadata(docs::tags = ...))] to the enum variant", + name + ); + continue; + } }; entries.push(MetricEntry { From 52690197bc155ec5ec302bd752d1918caa22e3a2 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 15:09:27 -0400 Subject: [PATCH 33/43] fix(metrics): add docs::tags annotations to all 90 remaining unannotated variants --- .../src/internal_event/metric_name.rs | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index d7e4d9200ce49..aab1b747145ae 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -59,40 +59,52 @@ pub enum CounterName { ComponentErrorsTotal, /// The total number of events for which this source responded with a timeout error. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ComponentTimedOutEventsTotal, /// The total number of requests for which this source responded with a timeout error. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ComponentTimedOutRequestsTotal, /// The number of events received by this buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferReceivedEventsTotal, /// The number of bytes received by this buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferReceivedBytesTotal, /// The number of events sent by this buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferSentEventsTotal, /// The number of bytes sent by this buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferSentBytesTotal, /// The number of events dropped by this non-blocking buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferDiscardedEventsTotal, /// The number of bytes dropped by this non-blocking buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferDiscardedBytesTotal, /// The total number of buffer errors encountered. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferErrorsTotal, // Internal events from src/internal_events/ /// The number of events recorded by the aggregate transform. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AggregateEventsRecordedTotal, /// The number of failed metric updates, `incremental` adds, encountered by the aggregate transform. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AggregateFailedUpdates, /// The number of flushes done by the aggregate transform. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AggregateFlushesTotal, /// The number of times the Vector API has been started. @@ -112,6 +124,7 @@ pub enum CounterName { CollectCompletedTotal, /// The total number of times a command has been executed. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] CommandExecutedTotal, /// The total number of times a connection has been established. @@ -127,30 +140,39 @@ pub enum CounterName { ConnectionShutdownTotal, /// The total number of container events processed. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ContainerProcessedEventsTotal, /// The total number of times Vector stopped watching for container logs. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ContainersUnwatchedTotal, /// The total number of times Vector started watching for container logs. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ContainersWatchedTotal, /// The total number of byte order marks (BOM) removed from incoming data. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] DecoderBomRemovalsTotal, /// The total number of warnings when replacing malformed characters during decoding. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] DecoderMalformedReplacementWarningsTotal, /// The total number of bytes loaded into Doris. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] DorisBytesLoadedTotal, /// The total number of rows filtered by Doris during stream load. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] DorisRowsFilteredTotal, /// The total number of rows successfully loaded into Doris. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] DorisRowsLoadedTotal, /// The total number of warnings when replacing unmappable characters during encoding. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] EncoderUnmappableReplacementWarningsTotal, /// The total number of events discarded by this component. @@ -182,6 +204,7 @@ pub enum CounterName { GrpcServerMessagesSentTotal, /// The total number of HTTP client errors encountered. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] HttpClientErrorsTotal, /// The total number of sent HTTP requests, tagged with the request method. @@ -201,36 +224,47 @@ pub enum CounterName { HttpServerResponsesSentTotal, /// Total number of message bytes (including framing) received from Kafka brokers. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaConsumedMessagesBytesTotal, /// Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaConsumedMessagesTotal, /// Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaProducedMessagesBytesTotal, /// Total number of messages transmitted (produced) to Kafka brokers. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaProducedMessagesTotal, /// Total number of bytes transmitted to Kafka brokers. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaRequestsBytesTotal, /// Total number of requests sent to Kafka brokers. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaRequestsTotal, /// Total number of bytes received from Kafka brokers. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaResponsesBytesTotal, /// Total number of responses received from Kafka brokers. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaResponsesTotal, /// The total number of failed efforts to refresh AWS EC2 metadata. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MetadataRefreshFailedTotal, /// The total number of AWS EC2 metadata refreshes. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MetadataRefreshSuccessfulTotal, /// The total number of errors encountered while parsing. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ParseErrorsTotal, /// The total number of times the Vector instance has quit. @@ -242,24 +276,31 @@ pub enum CounterName { ReloadedTotal, /// The total number of events with rewrapped timestamps. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] RewrittenTimestampEventsTotal, /// The total number of successful deferrals of SQS messages. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] SqsMessageDeferSucceededTotal, /// The total number of successful deletions of SQS messages. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] SqsMessageDeleteSucceededTotal, /// The total number of SQS messages successfully processed. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] SqsMessageProcessingSucceededTotal, /// The total number of times successfully receiving SQS messages. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] SqsMessageReceiveSucceededTotal, /// The total number of received SQS messages. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] SqsMessageReceivedMessagesTotal, /// The number of stale events that Vector has flushed. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] StaleEventsFlushedTotal, /// The total number of times the Vector instance has been started. @@ -271,6 +312,7 @@ pub enum CounterName { StoppedTotal, /// The total number of events that contained a tag which exceeded the configured cardinality limit. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] TagCardinalityUntrackedEventsTotal, /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. @@ -281,12 +323,15 @@ pub enum CounterName { TagValueLimitExceededTotal, /// The total number of times the value limit was reached. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ValueLimitReachedTotal, /// The total number of bytes sent over WebSocket connections. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] WebsocketBytesSentTotal, /// The total number of messages sent over WebSocket connections. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] WebsocketMessagesSentTotal, /// The total number of times the Windows service has been installed. @@ -310,15 +355,19 @@ pub enum CounterName { WindowsServiceUninstallTotal, /// The total number of failures to annotate Kubernetes events with namespace metadata. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] K8sEventNamespaceAnnotationFailuresTotal, /// The total number of failures to annotate Kubernetes events with node metadata. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] K8sEventNodeAnnotationFailuresTotal, /// The total number of edge cases encountered while picking format of the Kubernetes log message. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] K8sFormatPickerEdgeCasesTotal, /// The total number of failures to parse a message as a JSON object. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] K8sDockerFormatParseFailuresTotal, /// The total number of times an S3 record in an SQS message was ignored. @@ -332,27 +381,35 @@ pub enum CounterName { SqsS3EventRecordIgnoredTotal, /// The total number of bytes allocated by this component. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ComponentAllocatedBytesTotal, /// The total number of bytes deallocated by this component. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ComponentDeallocatedBytesTotal, /// The total number of failed insertions into the in-memory enrichment table. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MemoryEnrichmentTableFailedInsertions, /// The total number of failed reads from the in-memory enrichment table. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MemoryEnrichmentTableFailedReads, /// The total number of flushes of the in-memory enrichment table. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MemoryEnrichmentTableFlushesTotal, /// The total number of successful insertions into the in-memory enrichment table. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MemoryEnrichmentTableInsertionsTotal, /// The total number of successful reads from the in-memory enrichment table. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MemoryEnrichmentTableReadsTotal, /// The total number of entries evicted from the in-memory enrichment table due to TTL expiration. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MemoryEnrichmentTableTtlExpirations, /// The total number of errors reading datagram. @@ -394,6 +451,7 @@ pub enum HistogramName { ComponentReceivedBytes, /// The duration spent sending a payload to this buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferSendDurationSeconds, /// The elapsed time, in fractional seconds, that an event spends in a single transform. @@ -404,33 +462,43 @@ pub enum HistogramName { ComponentLatencySeconds, /// The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] SourceLagTimeSeconds, /// The time elapsed blocking on the downstream channel to accept a single chunk from a batch of events received at the source. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] SourceSendLatencySeconds, /// The time elapsed blocking on the downstream channel to accept an entire batch of events received at the source. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] SourceSendBatchLatencySeconds, /// The average round-trip time (RTT) for the current window. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyAveragedRtt, /// The amount of back pressure on the current component. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyBackPressure, /// The number of outbound requests currently awaiting a response. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyInFlight, /// The concurrency limit that the adaptive concurrency feature has decided on for this current window. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyLimit, /// The observed round-trip time (RTT) for requests. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyObservedRtt, /// The mean round-trip time (RTT) for the current window. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyPastRttMean, /// The number of times the concurrency limit was reached. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyReachedLimit, /// The time taken to process an S3 object that succeeded, in seconds. @@ -446,6 +514,7 @@ pub enum HistogramName { CollectDurationSeconds, /// The command execution duration in seconds. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] CommandExecutionDurationSeconds, /// The duration spent handling a gRPC request. @@ -457,6 +526,7 @@ pub enum HistogramName { HttpServerHandlerDurationSeconds, /// The round-trip time (RTT) of HTTP requests. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] HttpClientRttSeconds, /// The round-trip time (RTT) of HTTP requests, tagged with the response code. @@ -464,6 +534,7 @@ pub enum HistogramName { HttpClientResponseRttSeconds, /// The round-trip time (RTT) of HTTP requests that resulted in an error. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] HttpClientErrorRttSeconds, /// The utilization of the source buffer. @@ -587,40 +658,51 @@ pub enum GaugeName { TransformBufferUtilizationMean, /// The maximum number of events in the buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferMaxSizeEvents, /// The maximum size in events that the buffer can store. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferMaxEventSize, /// The maximum number of bytes in the buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferMaxSizeBytes, /// The maximum size in bytes that the buffer can store. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferMaxByteSize, /// The number of events currently in the buffer. #[configurable( deprecated = "This metric has been deprecated in favor of `buffer_size_events`." )] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferEvents, /// The number of events currently in the buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferSizeEvents, /// The number of bytes currently in the buffer. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferSizeBytes, /// The number of bytes currently in the buffer. #[configurable(deprecated = "This metric has been deprecated in favor of `buffer_size_bytes`.")] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferByteSize, /// The current utilization of this component, expressed as a value from 0 to 1. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] Utilization, /// The number of bytes currently allocated by this component. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ComponentAllocatedBytes, /// The total number of open files. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] OpenFiles, /// The number of seconds the Vector instance has been running. @@ -638,9 +720,11 @@ pub enum GaugeName { BuildInfo, /// Current number of messages in producer queues. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaQueueMessages, /// Current total size of messages in producer queues. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] KafkaQueueMessagesBytes, /// The Kafka consumer lag. @@ -659,21 +743,27 @@ pub enum GaugeName { OpenConnections, /// The number of currently active endpoints. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ActiveEndpoints, /// The number of outstanding Splunk HEC indexer acknowledgement acks. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] SplunkPendingAcks, /// Number of clients attached to a component. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ActiveClients, /// The number of objects currently stored in the in-memory enrichment table. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MemoryEnrichmentTableObjectsCount, /// The total size in bytes of all objects stored in the in-memory enrichment table. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] MemoryEnrichmentTableByteSize, /// The number of tag keys currently being tracked by the tag cardinality limit transform. + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] TagCardinalityTrackedKeys, /// The total number of metrics emitted from the internal metrics registry. From ac0a975b0712d04a11569ca1a4b12c55d7d6d3e6 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 15:16:18 -0400 Subject: [PATCH 34/43] fix(metrics): remove ghost metric CUE references, fix websocket_server to use generated metrics --- website/cue/reference/components/sinks.cue | 2 - .../cue/reference/components/sinks/loki.cue | 1 - .../cue/reference/components/sinks/nats.cue | 1 - .../cue/reference/components/sinks/redis.cue | 1 - .../cue/reference/components/sinks/vector.cue | 1 - .../components/sinks/websocket_server.cue | 14 +- .../reference/components/sources/stdin.cue | 1 - .../reference/components/sources/vector.cue | 1 - .../internal_metric_descriptions.cue | 192 +++++++++--------- 9 files changed, 98 insertions(+), 116 deletions(-) diff --git a/website/cue/reference/components/sinks.cue b/website/cue/reference/components/sinks.cue index 416df6cdb280d..a3c9c9b9be9b2 100644 --- a/website/cue/reference/components/sinks.cue +++ b/website/cue/reference/components/sinks.cue @@ -667,9 +667,7 @@ components: sinks: [Name=string]: { buffer_size_events: components.sources.internal_metrics.output.metrics.buffer_size_events buffer_events: components.sources.internal_metrics.output.metrics.buffer_events buffer_received_events_total: components.sources.internal_metrics.output.metrics.buffer_received_events_total - buffer_received_event_bytes_total: components.sources.internal_metrics.output.metrics.buffer_received_event_bytes_total buffer_sent_events_total: components.sources.internal_metrics.output.metrics.buffer_sent_events_total - buffer_sent_event_bytes_total: components.sources.internal_metrics.output.metrics.buffer_sent_event_bytes_total component_discarded_events_total: components.sources.internal_metrics.output.metrics.component_discarded_events_total component_errors_total: components.sources.internal_metrics.output.metrics.component_errors_total component_received_events_count: components.sources.internal_metrics.output.metrics.component_received_events_count diff --git a/website/cue/reference/components/sinks/loki.cue b/website/cue/reference/components/sinks/loki.cue index a1d245016178b..3f862e484001f 100644 --- a/website/cue/reference/components/sinks/loki.cue +++ b/website/cue/reference/components/sinks/loki.cue @@ -159,6 +159,5 @@ components: sinks: loki: { } telemetry: metrics: { - streams_total: components.sources.internal_metrics.output.metrics.streams_total } } diff --git a/website/cue/reference/components/sinks/nats.cue b/website/cue/reference/components/sinks/nats.cue index 99c101d5d8665..2c23cc175b6c6 100644 --- a/website/cue/reference/components/sinks/nats.cue +++ b/website/cue/reference/components/sinks/nats.cue @@ -64,6 +64,5 @@ components: sinks: nats: { how_it_works: components._nats.how_it_works telemetry: metrics: { - send_errors_total: components.sources.internal_metrics.output.metrics.send_errors_total } } diff --git a/website/cue/reference/components/sinks/redis.cue b/website/cue/reference/components/sinks/redis.cue index 0ba92f88e56ea..7708db0c9fde4 100644 --- a/website/cue/reference/components/sinks/redis.cue +++ b/website/cue/reference/components/sinks/redis.cue @@ -82,6 +82,5 @@ components: sinks: redis: { } telemetry: metrics: { - send_errors_total: components.sources.internal_metrics.output.metrics.send_errors_total } } diff --git a/website/cue/reference/components/sinks/vector.cue b/website/cue/reference/components/sinks/vector.cue index bcffbf3ba0114..dae2f57d4c044 100644 --- a/website/cue/reference/components/sinks/vector.cue +++ b/website/cue/reference/components/sinks/vector.cue @@ -81,6 +81,5 @@ components: sinks: vector: { how_it_works: components.sinks.vector.how_it_works telemetry: metrics: { - protobuf_decode_errors_total: components.sources.internal_metrics.output.metrics.protobuf_decode_errors_total } } diff --git a/website/cue/reference/components/sinks/websocket_server.cue b/website/cue/reference/components/sinks/websocket_server.cue index ac4426f05802c..feb1279025410 100644 --- a/website/cue/reference/components/sinks/websocket_server.cue +++ b/website/cue/reference/components/sinks/websocket_server.cue @@ -86,18 +86,8 @@ components: sinks: websocket_server: { open_connections: components.sources.internal_metrics.output.metrics.open_connections connection_established_total: components.sources.internal_metrics.output.metrics.connection_established_total connection_shutdown_total: components.sources.internal_metrics.output.metrics.connection_shutdown_total - websocket_messages_sent_total: { - description: "Number of messages sent from the websocket server." - type: "counter" - default_namespace: "vector" - tags: components.sources.internal_metrics.output.metrics._component_tags - } - websocket_bytes_sent_total: { - description: "Bytes sent from the websocket server." - type: "counter" - default_namespace: "vector" - tags: components.sources.internal_metrics.output.metrics._component_tags - } + websocket_messages_sent_total: components.sources.internal_metrics.output.metrics.websocket_messages_sent_total + websocket_bytes_sent_total: components.sources.internal_metrics.output.metrics.websocket_bytes_sent_total } configuration: generated.components.sinks.websocket_server.configuration diff --git a/website/cue/reference/components/sources/stdin.cue b/website/cue/reference/components/sources/stdin.cue index 262289538aac7..693cd0d5ae655 100644 --- a/website/cue/reference/components/sources/stdin.cue +++ b/website/cue/reference/components/sources/stdin.cue @@ -93,6 +93,5 @@ components: sources: stdin: { } telemetry: metrics: { - stdin_reads_failed_total: components.sources.internal_metrics.output.metrics.stdin_reads_failed_total } } diff --git a/website/cue/reference/components/sources/vector.cue b/website/cue/reference/components/sources/vector.cue index 3f7c18cef245b..42e1f8b23c7f4 100644 --- a/website/cue/reference/components/sources/vector.cue +++ b/website/cue/reference/components/sources/vector.cue @@ -106,6 +106,5 @@ components: sources: vector: { grpc_server_handler_duration_seconds: components.sources.internal_metrics.output.metrics.grpc_server_handler_duration_seconds grpc_server_messages_received_total: components.sources.internal_metrics.output.metrics.grpc_server_messages_received_total grpc_server_messages_sent_total: components.sources.internal_metrics.output.metrics.grpc_server_messages_sent_total - protobuf_decode_errors_total: components.sources.internal_metrics.output.metrics.protobuf_decode_errors_total } } diff --git a/website/cue/reference/generated/internal_metric_descriptions.cue b/website/cue/reference/generated/internal_metric_descriptions.cue index 136a0fca5f6cd..ad4ffa8233dd5 100644 --- a/website/cue/reference/generated/internal_metric_descriptions.cue +++ b/website/cue/reference/generated/internal_metric_descriptions.cue @@ -10,73 +10,73 @@ components: sources: internal_metrics: output: metrics: { description: "Number of clients attached to a component." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } active_endpoints: { description: "The number of currently active endpoints." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } adaptive_concurrency_averaged_rtt: { description: "The average round-trip time (RTT) for the current window." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } adaptive_concurrency_back_pressure: { description: "The amount of back pressure on the current component." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } adaptive_concurrency_in_flight: { description: "The number of outbound requests currently awaiting a response." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } adaptive_concurrency_limit: { description: "The concurrency limit that the adaptive concurrency feature has decided on for this current window." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } adaptive_concurrency_observed_rtt: { description: "The observed round-trip time (RTT) for requests." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } adaptive_concurrency_past_rtt_mean: { description: "The mean round-trip time (RTT) for the current window." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } adaptive_concurrency_reached_limit: { description: "The number of times the concurrency limit was reached." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } aggregate_events_recorded_total: { description: "The number of events recorded by the aggregate transform." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } aggregate_failed_updates: { description: "The number of failed metric updates, `incremental` adds, encountered by the aggregate transform." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } aggregate_flushes_total: { description: "The number of flushes done by the aggregate transform." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } api_started_total: { description: "The number of times the Vector API has been started." @@ -85,10 +85,10 @@ components: sources: internal_metrics: output: metrics: { tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} } buffer_byte_size: { - description: "The number of bytes currently in the buffer." - type: "gauge" - default_namespace: "vector" - tags: _component_tags + description: "The number of bytes currently in the buffer." + type: "gauge" + default_namespace: "vector" + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} deprecated: true deprecated_message: "This metric has been deprecated in favor of `buffer_size_bytes`." } @@ -96,25 +96,25 @@ components: sources: internal_metrics: output: metrics: { description: "The number of bytes dropped by this non-blocking buffer." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_discarded_events_total: { description: "The number of events dropped by this non-blocking buffer." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_errors_total: { description: "The total number of buffer errors encountered." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_events: { - description: "The number of events currently in the buffer." - type: "gauge" - default_namespace: "vector" - tags: _component_tags + description: "The number of events currently in the buffer." + type: "gauge" + default_namespace: "vector" + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} deprecated: true deprecated_message: "This metric has been deprecated in favor of `buffer_size_events`." } @@ -122,67 +122,67 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum size in bytes that the buffer can store." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_max_event_size: { description: "The maximum size in events that the buffer can store." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_max_size_bytes: { description: "The maximum number of bytes in the buffer." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_max_size_events: { description: "The maximum number of events in the buffer." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_received_bytes_total: { description: "The number of bytes received by this buffer." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_received_events_total: { description: "The number of events received by this buffer." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_send_duration_seconds: { description: "The duration spent sending a payload to this buffer." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_sent_bytes_total: { description: "The number of bytes sent by this buffer." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_sent_events_total: { description: "The number of events sent by this buffer." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_size_bytes: { description: "The number of bytes currently in the buffer." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } buffer_size_events: { description: "The number of events currently in the buffer." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } build_info: { description: "Pseudo-metric that provides build information for the Vector instance." @@ -218,31 +218,31 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of times a command has been executed." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } command_execution_duration_seconds: { description: "The command execution duration in seconds." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } component_allocated_bytes: { description: "The number of bytes currently allocated by this component." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } component_allocated_bytes_total: { description: "The total number of bytes allocated by this component." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } component_deallocated_bytes_total: { description: "The total number of bytes deallocated by this component." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } component_discarded_events_total: { description: "The number of events dropped by this component." @@ -320,13 +320,13 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of events for which this source responded with a timeout error." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } component_timed_out_requests_total: { description: "The total number of requests for which this source responded with a timeout error." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } config_reload_rejected: { description: "Number of configuration reload attempts that were rejected." @@ -362,55 +362,55 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of container events processed." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } containers_unwatched_total: { description: "The total number of times Vector stopped watching for container logs." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } containers_watched_total: { description: "The total number of times Vector started watching for container logs." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } decoder_bom_removals_total: { description: "The total number of byte order marks (BOM) removed from incoming data." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } decoder_malformed_replacement_warnings_total: { description: "The total number of warnings when replacing malformed characters during decoding." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } doris_bytes_loaded_total: { description: "The total number of bytes loaded into Doris." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } doris_rows_filtered_total: { description: "The total number of rows filtered by Doris during stream load." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } doris_rows_loaded_total: { description: "The total number of rows successfully loaded into Doris." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } encoder_unmappable_replacement_warnings_total: { description: "The total number of warnings when replacing unmappable characters during encoding." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } events_discarded_total: { description: "The total number of events discarded by this component." @@ -464,13 +464,13 @@ components: sources: internal_metrics: output: metrics: { description: "The round-trip time (RTT) of HTTP requests that resulted in an error." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } http_client_errors_total: { description: "The total number of HTTP client errors encountered." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } http_client_requests_sent_total: { description: "The total number of sent HTTP requests, tagged with the request method." @@ -494,7 +494,7 @@ components: sources: internal_metrics: output: metrics: { description: "The round-trip time (RTT) of HTTP requests." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } http_server_handler_duration_seconds: { description: "The duration spent handling an HTTP request." @@ -530,37 +530,37 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of failures to parse a message as a JSON object." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } k8s_event_namespace_annotation_failures_total: { description: "The total number of failures to annotate Kubernetes events with namespace metadata." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } k8s_event_node_annotation_failures_total: { description: "The total number of failures to annotate Kubernetes events with node metadata." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } k8s_format_picker_edge_cases_total: { description: "The total number of edge cases encountered while picking format of the Kubernetes log message." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_consumed_messages_bytes_total: { description: "Total number of message bytes (including framing) received from Kafka brokers." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_consumed_messages_total: { description: "Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_consumer_lag: { description: "The Kafka consumer lag." @@ -572,49 +572,49 @@ components: sources: internal_metrics: output: metrics: { description: "Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_produced_messages_total: { description: "Total number of messages transmitted (produced) to Kafka brokers." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_queue_messages: { description: "Current number of messages in producer queues." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_queue_messages_bytes: { description: "Current total size of messages in producer queues." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_requests_bytes_total: { description: "Total number of bytes transmitted to Kafka brokers." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_requests_total: { description: "Total number of requests sent to Kafka brokers." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_responses_bytes_total: { description: "Total number of bytes received from Kafka brokers." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } kafka_responses_total: { description: "Total number of responses received from Kafka brokers." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } lua_memory_used_bytes: { description: "The total memory currently being used by the Lua runtime." @@ -626,61 +626,61 @@ components: sources: internal_metrics: output: metrics: { description: "The total size in bytes of all objects stored in the in-memory enrichment table." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } memory_enrichment_table_failed_insertions: { description: "The total number of failed insertions into the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } memory_enrichment_table_failed_reads: { description: "The total number of failed reads from the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } memory_enrichment_table_flushes_total: { description: "The total number of flushes of the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } memory_enrichment_table_insertions_total: { description: "The total number of successful insertions into the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } memory_enrichment_table_objects_count: { description: "The number of objects currently stored in the in-memory enrichment table." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } memory_enrichment_table_reads_total: { description: "The total number of successful reads from the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } memory_enrichment_table_ttl_expirations: { description: "The total number of entries evicted from the in-memory enrichment table due to TTL expiration." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } metadata_refresh_failed_total: { description: "The total number of failed efforts to refresh AWS EC2 metadata." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } metadata_refresh_successful_total: { description: "The total number of AWS EC2 metadata refreshes." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } open_connections: { description: "The number of current open connections to Vector." @@ -692,13 +692,13 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of open files." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } parse_errors_total: { description: "The total number of errors encountered while parsing." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } quit_total: { description: "The total number of times the Vector instance has quit." @@ -716,7 +716,7 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of events with rewrapped timestamps." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } s3_object_processing_failed_duration_seconds: { description: "The time taken to process an S3 object that failed, in seconds." @@ -780,55 +780,55 @@ components: sources: internal_metrics: output: metrics: { description: "The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } source_send_batch_latency_seconds: { description: "The time elapsed blocking on the downstream channel to accept an entire batch of events received at the source." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } source_send_latency_seconds: { description: "The time elapsed blocking on the downstream channel to accept a single chunk from a batch of events received at the source." type: "histogram" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } splunk_pending_acks: { description: "The number of outstanding Splunk HEC indexer acknowledgement acks." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } sqs_message_defer_succeeded_total: { description: "The total number of successful deferrals of SQS messages." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } sqs_message_delete_succeeded_total: { description: "The total number of successful deletions of SQS messages." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } sqs_message_processing_succeeded_total: { description: "The total number of SQS messages successfully processed." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } sqs_message_receive_succeeded_total: { description: "The total number of times successfully receiving SQS messages." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } sqs_message_received_messages_total: { description: "The total number of received SQS messages." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } sqs_s3_event_record_ignored_total: { description: "The total number of times an S3 record in an SQS message was ignored." @@ -840,7 +840,7 @@ components: sources: internal_metrics: output: metrics: { description: "The number of stale events that Vector has flushed." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } started_total: { description: "The total number of times the Vector instance has been started." @@ -858,13 +858,13 @@ components: sources: internal_metrics: output: metrics: { description: "The number of tag keys currently being tracked by the tag cardinality limit transform." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } tag_cardinality_untracked_events_total: { description: "The total number of events that contained a tag which exceeded the configured cardinality limit." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } tag_value_limit_exceeded_total: { description: "The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`." @@ -934,25 +934,25 @@ components: sources: internal_metrics: output: metrics: { description: "The current utilization of this component, expressed as a value from 0 to 1." type: "gauge" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } value_limit_reached_total: { description: "The total number of times the value limit was reached." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } websocket_bytes_sent_total: { description: "The total number of bytes sent over WebSocket connections." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } websocket_messages_sent_total: { description: "The total number of messages sent over WebSocket connections." type: "counter" default_namespace: "vector" - tags: _component_tags + tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} } windows_service_install_total: { description: "The total number of times the Windows service has been installed." From 0de82202f61f12917cad98531b827d1a9ddaf215 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 15:24:40 -0400 Subject: [PATCH 35/43] fmt cue --- website/cue/reference/components/sinks/loki.cue | 3 +-- website/cue/reference/components/sinks/nats.cue | 3 +-- website/cue/reference/components/sinks/redis.cue | 3 +-- website/cue/reference/components/sinks/vector.cue | 3 +-- .../cue/reference/components/sinks/websocket_server.cue | 8 ++++---- website/cue/reference/components/sources/stdin.cue | 3 +-- 6 files changed, 9 insertions(+), 14 deletions(-) diff --git a/website/cue/reference/components/sinks/loki.cue b/website/cue/reference/components/sinks/loki.cue index 3f862e484001f..030ecefe35e84 100644 --- a/website/cue/reference/components/sinks/loki.cue +++ b/website/cue/reference/components/sinks/loki.cue @@ -158,6 +158,5 @@ components: sinks: loki: { } } - telemetry: metrics: { - } + telemetry: metrics: {} } diff --git a/website/cue/reference/components/sinks/nats.cue b/website/cue/reference/components/sinks/nats.cue index 2c23cc175b6c6..e2bc37d766aae 100644 --- a/website/cue/reference/components/sinks/nats.cue +++ b/website/cue/reference/components/sinks/nats.cue @@ -63,6 +63,5 @@ components: sinks: nats: { how_it_works: components._nats.how_it_works - telemetry: metrics: { - } + telemetry: metrics: {} } diff --git a/website/cue/reference/components/sinks/redis.cue b/website/cue/reference/components/sinks/redis.cue index 7708db0c9fde4..0de99a896ac04 100644 --- a/website/cue/reference/components/sinks/redis.cue +++ b/website/cue/reference/components/sinks/redis.cue @@ -81,6 +81,5 @@ components: sinks: redis: { } } - telemetry: metrics: { - } + telemetry: metrics: {} } diff --git a/website/cue/reference/components/sinks/vector.cue b/website/cue/reference/components/sinks/vector.cue index dae2f57d4c044..ec797e262960a 100644 --- a/website/cue/reference/components/sinks/vector.cue +++ b/website/cue/reference/components/sinks/vector.cue @@ -80,6 +80,5 @@ components: sinks: vector: { how_it_works: components.sinks.vector.how_it_works - telemetry: metrics: { - } + telemetry: metrics: {} } diff --git a/website/cue/reference/components/sinks/websocket_server.cue b/website/cue/reference/components/sinks/websocket_server.cue index feb1279025410..fc8273640603e 100644 --- a/website/cue/reference/components/sinks/websocket_server.cue +++ b/website/cue/reference/components/sinks/websocket_server.cue @@ -82,10 +82,10 @@ components: sinks: websocket_server: { } telemetry: metrics: { - active_clients: components.sources.internal_metrics.output.metrics.active_clients - open_connections: components.sources.internal_metrics.output.metrics.open_connections - connection_established_total: components.sources.internal_metrics.output.metrics.connection_established_total - connection_shutdown_total: components.sources.internal_metrics.output.metrics.connection_shutdown_total + active_clients: components.sources.internal_metrics.output.metrics.active_clients + open_connections: components.sources.internal_metrics.output.metrics.open_connections + connection_established_total: components.sources.internal_metrics.output.metrics.connection_established_total + connection_shutdown_total: components.sources.internal_metrics.output.metrics.connection_shutdown_total websocket_messages_sent_total: components.sources.internal_metrics.output.metrics.websocket_messages_sent_total websocket_bytes_sent_total: components.sources.internal_metrics.output.metrics.websocket_bytes_sent_total } diff --git a/website/cue/reference/components/sources/stdin.cue b/website/cue/reference/components/sources/stdin.cue index 693cd0d5ae655..b4b5c8aa6456f 100644 --- a/website/cue/reference/components/sources/stdin.cue +++ b/website/cue/reference/components/sources/stdin.cue @@ -92,6 +92,5 @@ components: sources: stdin: { } } - telemetry: metrics: { - } + telemetry: metrics: {} } From 164b216abd5ad8c86f0c15e6c93815c0b79b8fa5 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 16:04:25 -0400 Subject: [PATCH 36/43] Remove json_tags_to_cue, add examples, and use cue import --- .../src/internal_event/metric_tags.rs | 8 +- .../commands/build/component_docs/runner.rs | 212 +- .../internal_metric_descriptions.cue | 5343 ++++++++++++++++- 3 files changed, 5265 insertions(+), 298 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index 4b5e3a3255782..8a1de70f5caa1 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -19,8 +19,8 @@ pub fn merge_lazy(base: &LazyLock, extra: Value) -> Value { pub static INTERNAL_METRICS_TAGS: LazyLock = LazyLock::new(|| { json!({ - "pid": {"description": "The process ID of the Vector instance.", "required": false}, - "host": {"description": "The hostname of the system Vector is running on.", "required": false} + "pid": {"description": "The process ID of the Vector instance.", "required": false, "examples": ["4232"]}, + "host": {"description": "The hostname of the system Vector is running on.", "required": false, "examples": ["my-host.local"]} }) }); @@ -37,8 +37,8 @@ pub static COMPONENT_TAGS: LazyLock = LazyLock::new(|| { "transform": "Vector transform components" } }, - "component_id": {"description": "The Vector component ID.", "required": true}, - "component_type": {"description": "The Vector component type.", "required": true} + "component_id": {"description": "The Vector component ID.", "required": true, "examples": ["my_source", "my_sink"]}, + "component_type": {"description": "The Vector component type.", "required": true, "examples": ["file", "http", "honeycomb", "splunk_hec"]} }), ) }); diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index bd5002d07f17b..7e0977cec10a2 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -2,7 +2,6 @@ use super::schema::SchemaContext; use anyhow::{Context, Result, bail}; use indexmap::IndexMap; use serde_json::{Value, json}; -use std::fmt::Write as _; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -75,7 +74,7 @@ pub fn run(schema_path: &Path) -> Result<()> { // 4. Generate internal_metric_descriptions.cue from the metric name enums // injected into the schema by `vector generate-schema`. if let Some(metric_schemas) = root_schema.get("_metric_schemas") { - generate_internal_metric_descriptions(metric_schemas)?; + generate_internal_metric_descriptions(&mut context, metric_schemas)?; } Ok(()) @@ -362,80 +361,11 @@ fn render_and_import_generated_top_level_config_schema( Ok(()) } -/// Converts a flat JSON tag-field object to a CUE struct literal. -/// -/// The JSON is a map of tag-field names to field definitions: -/// ```json -/// { -/// "component_id": {"description": "...", "required": true}, -/// "output": {"description": "...", "required": false}, -/// "mode": {"description": "...", "required": true, -/// "enum": {"udp": "User Datagram Protocol"}} -/// } -/// ``` -/// Produces: `{component_id: {description: "...", required: true}, output: {...}, ...}` -fn json_tags_to_cue(obj: &serde_json::Map) -> String { - if obj.is_empty() { - return "{}".to_owned(); - } - - let fields: Vec = obj - .iter() - .map(|(name, field)| { - let field_cue = json_field_to_cue(field); - format!("{name}: {field_cue}") - }) - .collect(); - - format!("{{{}}}", fields.join(", ")) -} - -fn json_field_to_cue(val: &Value) -> String { - match val { - Value::Object(obj) => { - let desc = obj.get("description").and_then(Value::as_str).unwrap_or(""); - let required = obj - .get("required") - .and_then(Value::as_bool) - .unwrap_or(false); - let enum_part = obj - .get("enum") - .and_then(Value::as_object) - .map_or(String::new(), |e| { - let pairs: Vec = e - .iter() - .map(|(k, v)| format!("{k}: \"{}\"", v.as_str().unwrap_or_default())) - .collect(); - format!(", enum: {{{}}}", pairs.join(", ")) - }); - format!("{{description: \"{desc}\", required: {required}{enum_part}}}") - } - Value::String(s) => format!("\"{s}\""), - Value::Bool(b) => b.to_string(), - Value::Number(n) => n.to_string(), - _ => "{}".to_owned(), - } -} - -struct MetricEntry { - name: String, - metric_type: &'static str, - description: String, - tags: String, - deprecated: bool, - deprecated_message: Option, -} - -fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { - let out_path = - PathBuf::from("website/cue/reference/generated/internal_metric_descriptions.cue"); - - if let Some(parent) = out_path.parent() { - fs::create_dir_all(parent)?; - } - - let mut entries: Vec = Vec::new(); - +/// Collects all metric variants from `metric_schemas` into an alphabetically-sorted JSON map. +fn collect_metric_entries( + metric_schemas: &Value, +) -> std::collections::BTreeMap { + let mut metrics = std::collections::BTreeMap::new(); for (metric_type, schema) in [ ("counter", metric_schemas.get("counters")), ("histogram", metric_schemas.get("histograms")), @@ -454,19 +384,10 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { let Some(desc) = variant.get("description").and_then(Value::as_str) else { continue; }; - let deprecated = variant - .get("deprecated") - .and_then(Value::as_bool) - .unwrap_or(false); - let deprecated_message = variant - .get("_metadata") - .and_then(|m| m.get("deprecated_message")) - .and_then(Value::as_str) - .map(str::to_owned); let tags = match variant.get("_metadata").and_then(|m| m.get("docs::tags")) { - Some(Value::Object(obj)) => json_tags_to_cue(obj), - // String values are plain CUE expressions emitted verbatim (e.g. "{}"). - Some(Value::String(s)) => s.clone(), + Some(v @ Value::Object(_)) => v.clone(), + // "{}" is the only string value used — treat it as an empty object. + Some(Value::String(s)) if s == "{}" => json!({}), _ => { warn!( "metric variant '{}' has no docs::tags metadata — \ @@ -476,56 +397,91 @@ fn generate_internal_metric_descriptions(metric_schemas: &Value) -> Result<()> { continue; } }; - - entries.push(MetricEntry { - name: name.to_owned(), - metric_type, - description: desc - .replace('\\', "\\\\") - .replace('"', "\\\"") - .replace('\n', "\\n"), - tags, - deprecated, - deprecated_message, + let deprecated = variant + .get("deprecated") + .and_then(Value::as_bool) + .unwrap_or(false); + let mut entry = json!({ + "description": desc, + "type": metric_type, + "default_namespace": "vector", + "tags": tags, }); + if deprecated { + entry["deprecated"] = json!(true); + if let Some(msg) = variant + .get("_metadata") + .and_then(|m| m.get("deprecated_message")) + .and_then(Value::as_str) + { + entry["deprecated_message"] = json!(msg); + } + } + metrics.insert(name.to_owned(), entry); } } + metrics +} - entries.sort_by(|a, b| a.name.cmp(&b.name)); - - let mut cue = String::from( - "package metadata\n\ - \n\ - // Auto-generated by `vdev build component-docs`.\n\ - // Do not edit manually — update the doc comment (or #[configurable(deprecated)]\n\ - // attribute) on the relevant CounterName / HistogramName / GaugeName variant in\n\ - // lib/vector-common/src/internal_event/metric_name.rs instead,\n\ - // then re-run `make generate-component-docs`.\n\ - components: sources: internal_metrics: output: metrics: {\n", - ); +fn generate_internal_metric_descriptions( + context: &mut SchemaContext, + metric_schemas: &Value, +) -> Result<()> { + let out_path = + PathBuf::from("website/cue/reference/generated/internal_metric_descriptions.cue"); + + let metrics_value: serde_json::Map = + collect_metric_entries(metric_schemas).into_iter().collect(); - for e in &entries { - writeln!(cue, "\t{}: {{", e.name).unwrap(); - writeln!(cue, "\t\tdescription: \"{}\"", e.description).unwrap(); - writeln!(cue, "\t\ttype: \"{}\"", e.metric_type).unwrap(); - cue.push_str("\t\tdefault_namespace: \"vector\"\n"); - writeln!(cue, "\t\ttags: {}", e.tags).unwrap(); - if e.deprecated { - cue.push_str("\t\tdeprecated: true\n"); - if let Some(msg) = &e.deprecated_message { - let escaped = msg - .replace('\\', "\\\\") - .replace('"', "\\\"") - .replace('\n', "\\n"); - writeln!(cue, "\t\tdeprecated_message: \"{escaped}\"").unwrap(); + let data = json!({ + "components": { + "sources": { + "internal_metrics": { + "output": { + "metrics": Value::Object(metrics_value) + } + } } } - cue.push_str("\t}\n"); + }); + + let json_str = serde_json::to_string_pretty(&data)?; + let json_file = + write_to_temp_file("internal-metric-descriptions-", ".json", &json_str)?; + + if let Some(parent) = out_path.parent() { + fs::create_dir_all(parent)?; + } + + let status = Command::new(&context.cue_binary_path) + .args([ + "import", + "-f", + "-o", + out_path.to_str().unwrap(), + "-p", + "metadata", + json_file.to_str().unwrap(), + ]) + .status()?; + + if !status.success() { + bail!( + "Failed to import internal metric descriptions as valid CUE \ + (cue exit status {status}). JSON written to {json_path}.", + json_path = json_file.display() + ); } - cue.push_str("}\n"); + // Prepend the auto-generated header so editors know not to edit the file directly. + let generated = fs::read_to_string(&out_path)?; + let header = "// Auto-generated by `vdev build component-docs`.\n\ + // Do not edit manually — update the doc comment (or #[configurable(deprecated)]\n\ + // attribute) on the relevant CounterName / HistogramName / GaugeName variant in\n\ + // lib/vector-common/src/internal_event/metric_name.rs instead,\n\ + // then re-run `make generate-component-docs`.\n"; + fs::write(&out_path, format!("{header}{generated}"))?; - fs::write(&out_path, &cue)?; info!( "[✓] Wrote internal metric descriptions to '{}'.", out_path.display() diff --git a/website/cue/reference/generated/internal_metric_descriptions.cue b/website/cue/reference/generated/internal_metric_descriptions.cue index ad4ffa8233dd5..98b6809d844f7 100644 --- a/website/cue/reference/generated/internal_metric_descriptions.cue +++ b/website/cue/reference/generated/internal_metric_descriptions.cue @@ -1,94 +1,523 @@ -package metadata - // Auto-generated by `vdev build component-docs`. // Do not edit manually — update the doc comment (or #[configurable(deprecated)] // attribute) on the relevant CounterName / HistogramName / GaugeName variant in // lib/vector-common/src/internal_event/metric_name.rs instead, // then re-run `make generate-component-docs`. +package metadata + components: sources: internal_metrics: output: metrics: { active_clients: { description: "Number of clients attached to a component." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } active_endpoints: { description: "The number of currently active endpoints." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } adaptive_concurrency_averaged_rtt: { description: "The average round-trip time (RTT) for the current window." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } adaptive_concurrency_back_pressure: { description: "The amount of back pressure on the current component." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } adaptive_concurrency_in_flight: { description: "The number of outbound requests currently awaiting a response." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } adaptive_concurrency_limit: { description: "The concurrency limit that the adaptive concurrency feature has decided on for this current window." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } adaptive_concurrency_observed_rtt: { description: "The observed round-trip time (RTT) for requests." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } adaptive_concurrency_past_rtt_mean: { description: "The mean round-trip time (RTT) for the current window." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } adaptive_concurrency_reached_limit: { description: "The number of times the concurrency limit was reached." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } aggregate_events_recorded_total: { description: "The number of events recorded by the aggregate transform." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } aggregate_failed_updates: { description: "The number of failed metric updates, `incremental` adds, encountered by the aggregate transform." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } aggregate_flushes_total: { description: "The number of flushes done by the aggregate transform." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } api_started_total: { description: "The number of times the Vector API has been started." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } buffer_byte_size: { description: "The number of bytes currently in the buffer." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } deprecated: true deprecated_message: "This metric has been deprecated in favor of `buffer_size_bytes`." } @@ -96,25 +525,153 @@ components: sources: internal_metrics: output: metrics: { description: "The number of bytes dropped by this non-blocking buffer." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_discarded_events_total: { description: "The number of events dropped by this non-blocking buffer." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_errors_total: { description: "The total number of buffer errors encountered." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_events: { description: "The number of events currently in the buffer." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } deprecated: true deprecated_message: "This metric has been deprecated in favor of `buffer_size_events`." } @@ -122,397 +679,2589 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum size in bytes that the buffer can store." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_max_event_size: { description: "The maximum size in events that the buffer can store." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_max_size_bytes: { description: "The maximum number of bytes in the buffer." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_max_size_events: { description: "The maximum number of events in the buffer." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_received_bytes_total: { description: "The number of bytes received by this buffer." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_received_events_total: { description: "The number of events received by this buffer." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_send_duration_seconds: { description: "The duration spent sending a payload to this buffer." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_sent_bytes_total: { description: "The number of bytes sent by this buffer." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_sent_events_total: { description: "The number of events sent by this buffer." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_size_bytes: { description: "The number of bytes currently in the buffer." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } buffer_size_events: { description: "The number of events currently in the buffer." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } build_info: { description: "Pseudo-metric that provides build information for the Vector instance." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, debug: {description: "Whether this is a debug build of Vector", required: true}, version: {description: "Vector version.", required: true}, rust_version: {description: "The Rust version from the package manifest.", required: true}, arch: {description: "The target architecture being compiled for. (e.g. x86_64)", required: true}, revision: {description: "Revision identifer, related to versioned releases.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + debug: { + description: "Whether this is a debug build of Vector" + required: true + } + version: { + description: "Vector version." + required: true + } + rust_version: { + description: "The Rust version from the package manifest." + required: true + } + arch: { + description: "The target architecture being compiled for. (e.g. x86_64)" + required: true + } + revision: { + description: "Revision identifer, related to versioned releases." + required: true + } + } } checkpoints_total: { description: "The total number of files checkpointed." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } checksum_errors_total: { description: "The total number of errors identifying files via checksum." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + file: { + description: "The file that produced the error." + required: false + } + } } collect_completed_total: { description: "The total number of metrics collections completed for this component." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } collect_duration_seconds: { description: "The duration spent collecting metrics for this component." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } command_executed_total: { description: "The total number of times a command has been executed." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } command_execution_duration_seconds: { description: "The command execution duration in seconds." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } component_allocated_bytes: { description: "The number of bytes currently allocated by this component." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } component_allocated_bytes_total: { description: "The total number of bytes allocated by this component." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } component_deallocated_bytes_total: { description: "The total number of bytes deallocated by this component." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } component_discarded_events_total: { description: "The number of events dropped by this component." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, intentional: {description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + intentional: { + description: "True if the events were discarded intentionally, like a `filter` transform, or false if due to an error." + required: true + } + } } component_errors_total: { description: "The total number of errors encountered by this component." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, error_type: {description: "The type of the error", required: true, enum: {acknowledgements_failed: "The acknowledgement operation failed.", delete_failed: "The file deletion failed.", encode_failed: "The encode operation failed.", field_missing: "The event field was missing.", glob_failed: "The glob pattern match operation failed.", http_error: "The HTTP request resulted in an error code.", invalid_metric: "The metric was invalid.", kafka_offset_update: "The consumer offset update failed.", kafka_read: "The message from Kafka was invalid.", mapping_failed: "The mapping failed.", match_failed: "The match operation failed.", out_of_order: "The event was out of order.", parse_failed: "The parsing operation failed.", read_failed: "The file read operation failed.", render_error: "The rendering operation failed.", stream_closed: "The downstream was closed, forwarding the event(s) failed.", type_conversion_failed: "The type conversion operating failed.", type_field_does_not_exist: "The type field does not exist.", type_ip_address_parse_error: "The IP address did not parse.", unlabeled_event: "The event was not labeled.", value_invalid: "The value was invalid.", watch_failed: "The file watch operation failed.", write_failed: "The file write operation failed."}}, stage: {description: "The stage within the component at which the error occurred.", required: true, enum: {receiving: "While receiving data.", processing: "While processing data within the component.", sending: "While sending data."}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + error_type: { + description: "The type of the error" + required: true + enum: { + acknowledgements_failed: "The acknowledgement operation failed." + delete_failed: "The file deletion failed." + encode_failed: "The encode operation failed." + field_missing: "The event field was missing." + glob_failed: "The glob pattern match operation failed." + http_error: "The HTTP request resulted in an error code." + invalid_metric: "The metric was invalid." + kafka_offset_update: "The consumer offset update failed." + kafka_read: "The message from Kafka was invalid." + mapping_failed: "The mapping failed." + match_failed: "The match operation failed." + out_of_order: "The event was out of order." + parse_failed: "The parsing operation failed." + read_failed: "The file read operation failed." + render_error: "The rendering operation failed." + stream_closed: "The downstream was closed, forwarding the event(s) failed." + type_conversion_failed: "The type conversion operating failed." + type_field_does_not_exist: "The type field does not exist." + type_ip_address_parse_error: "The IP address did not parse." + unlabeled_event: "The event was not labeled." + value_invalid: "The value was invalid." + watch_failed: "The file watch operation failed." + write_failed: "The file write operation failed." + } + } + stage: { + description: "The stage within the component at which the error occurred." + required: true + enum: { + receiving: "While receiving data." + processing: "While processing data within the component." + sending: "While sending data." + } + } + } } component_latency_mean_seconds: { - description: "This includes both the time spent queued in the transform's input buffer and the time spent\nexecuting the transform itself. This value is smoothed over time using an exponentially\nweighted moving average (EWMA)." + description: """ + This includes both the time spent queued in the transform's input buffer and the time spent + executing the transform itself. This value is smoothed over time using an exponentially + weighted moving average (EWMA). + """ type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } component_latency_seconds: { - description: "This includes both the time spent queued in the transform's input buffer and the time spent\nexecuting the transform itself." + description: """ + This includes both the time spent queued in the transform's input buffer and the time spent + executing the transform itself. + """ type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } component_received_bytes: { description: "The size in bytes of each event received by the source." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + file: { + description: "The file from which the data originated." + required: false + } + uri: { + description: "The sanitized URI from which the data originated." + required: false + } + container_name: { + description: "The name of the container from which the data originated." + required: false + } + pod_name: { + description: "The name of the pod from which the data originated." + required: false + } + peer_addr: { + description: "The IP from which the data originated." + required: false + } + peer_path: { + description: "The pathname from which the data originated." + required: false + } + mode: { + description: "The connection mode used by the component." + required: false + enum: { + udp: "User Datagram Protocol" + tcp: "Transmission Control Protocol" + unix: "Unix domain socket" + } + } + } } component_received_bytes_total: { description: "The number of raw bytes accepted by this component from source origins." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + file: { + description: "The file from which the data originated." + required: false + } + uri: { + description: "The sanitized URI from which the data originated." + required: false + } + container_name: { + description: "The name of the container from which the data originated." + required: false + } + pod_name: { + description: "The name of the pod from which the data originated." + required: false + } + peer_addr: { + description: "The IP from which the data originated." + required: false + } + peer_path: { + description: "The pathname from which the data originated." + required: false + } + mode: { + description: "The connection mode used by the component." + required: false + enum: { + udp: "User Datagram Protocol" + tcp: "Transmission Control Protocol" + unix: "Unix domain socket" + } + } + } } component_received_event_bytes_total: { - description: "The number of event bytes accepted by this component either from\ntagged origins like file and uri, or cumulatively from other origins." + description: """ + The number of event bytes accepted by this component either from + tagged origins like file and uri, or cumulatively from other origins. + """ type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + file: { + description: "The file from which the data originated." + required: false + } + uri: { + description: "The sanitized URI from which the data originated." + required: false + } + container_name: { + description: "The name of the container from which the data originated." + required: false + } + pod_name: { + description: "The name of the pod from which the data originated." + required: false + } + peer_addr: { + description: "The IP from which the data originated." + required: false + } + peer_path: { + description: "The pathname from which the data originated." + required: false + } + mode: { + description: "The connection mode used by the component." + required: false + enum: { + udp: "User Datagram Protocol" + tcp: "Transmission Control Protocol" + unix: "Unix domain socket" + } + } + } } component_received_events_count: { - description: "Note that this is separate than sink-level batching. It is mostly useful for low level\ndebugging performance issues in Vector due to small internal batches." + description: """ + Note that this is separate than sink-level batching. It is mostly useful for low level + debugging performance issues in Vector due to small internal batches. + """ type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + file: { + description: "The file from which the data originated." + required: false + } + uri: { + description: "The sanitized URI from which the data originated." + required: false + } + container_name: { + description: "The name of the container from which the data originated." + required: false + } + pod_name: { + description: "The name of the pod from which the data originated." + required: false + } + peer_addr: { + description: "The IP from which the data originated." + required: false + } + peer_path: { + description: "The pathname from which the data originated." + required: false + } + mode: { + description: "The connection mode used by the component." + required: false + enum: { + udp: "User Datagram Protocol" + tcp: "Transmission Control Protocol" + unix: "Unix domain socket" + } + } + } } component_received_events_total: { - description: "The number of events accepted by this component either from tagged\norigins like file and uri, or cumulatively from other origins." + description: """ + The number of events accepted by this component either from tagged + origins like file and uri, or cumulatively from other origins. + """ type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, file: {description: "The file from which the data originated.", required: false}, uri: {description: "The sanitized URI from which the data originated.", required: false}, container_name: {description: "The name of the container from which the data originated.", required: false}, pod_name: {description: "The name of the pod from which the data originated.", required: false}, peer_addr: {description: "The IP from which the data originated.", required: false}, peer_path: {description: "The pathname from which the data originated.", required: false}, mode: {description: "The connection mode used by the component.", required: false, enum: {udp: "User Datagram Protocol", tcp: "Transmission Control Protocol", unix: "Unix domain socket"}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + file: { + description: "The file from which the data originated." + required: false + } + uri: { + description: "The sanitized URI from which the data originated." + required: false + } + container_name: { + description: "The name of the container from which the data originated." + required: false + } + pod_name: { + description: "The name of the pod from which the data originated." + required: false + } + peer_addr: { + description: "The IP from which the data originated." + required: false + } + peer_path: { + description: "The pathname from which the data originated." + required: false + } + mode: { + description: "The connection mode used by the component." + required: false + enum: { + udp: "User Datagram Protocol" + tcp: "Transmission Control Protocol" + unix: "Unix domain socket" + } + } + } } component_sent_bytes_total: { description: "The number of raw bytes sent by this component to destination sinks." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, endpoint: {description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string.", required: false}, file: {description: "The absolute path of the destination file.", required: false}, protocol: {description: "The protocol used to send the bytes.", required: true}, region: {description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + endpoint: { + description: "The endpoint to which the bytes were sent. For HTTP, this will be the host and path only, excluding the query string." + required: false + } + file: { + description: "The absolute path of the destination file." + required: false + } + protocol: { + description: "The protocol used to send the bytes." + required: true + } + region: { + description: "The AWS region name to which the bytes were sent. In some configurations, this may be a literal hostname." + required: false + } + } } component_sent_event_bytes_total: { description: "The total number of event bytes emitted by this component." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } component_sent_events_total: { description: "The total number of events emitted by this component." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } component_timed_out_events_total: { description: "The total number of events for which this source responded with a timeout error." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } component_timed_out_requests_total: { description: "The total number of requests for which this source responded with a timeout error." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } config_reload_rejected: { description: "Number of configuration reload attempts that were rejected." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, reason: {description: "The type of the error", required: true, enum: {out_of_order: "The event was out of order.", oversized: "The event was too large."}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + reason: { + description: "The type of the error" + required: true + enum: { + out_of_order: "The event was out of order." + oversized: "The event was too large." + } + } + } } connection_established_total: { description: "The total number of times a connection has been established." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } connection_read_errors_total: { description: "The total number of errors reading datagram." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, mode: {description: "", required: true, enum: {udp: "User Datagram Protocol"}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + mode: { + description: "" + required: true + enum: udp: "User Datagram Protocol" + } + } } connection_send_errors_total: { description: "The total number of errors sending data via the connection." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } connection_shutdown_total: { description: "The total number of times the connection has been shut down." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } container_processed_events_total: { description: "The total number of container events processed." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } containers_unwatched_total: { description: "The total number of times Vector stopped watching for container logs." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } containers_watched_total: { description: "The total number of times Vector started watching for container logs." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } decoder_bom_removals_total: { description: "The total number of byte order marks (BOM) removed from incoming data." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } decoder_malformed_replacement_warnings_total: { description: "The total number of warnings when replacing malformed characters during decoding." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } doris_bytes_loaded_total: { description: "The total number of bytes loaded into Doris." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } doris_rows_filtered_total: { description: "The total number of rows filtered by Doris during stream load." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } doris_rows_loaded_total: { description: "The total number of rows successfully loaded into Doris." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } encoder_unmappable_replacement_warnings_total: { description: "The total number of warnings when replacing unmappable characters during encoding." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } events_discarded_total: { description: "The total number of events discarded by this component." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, reason: {description: "The type of the error", required: true, enum: {out_of_order: "The event was out of order.", oversized: "The event was too large."}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + reason: { + description: "The type of the error" + required: true + enum: { + out_of_order: "The event was out of order." + oversized: "The event was too large." + } + } + } } files_added_total: { description: "The total number of files Vector has found to watch." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + file: { + description: "The file that produced the error." + required: false + } + } } files_deleted_total: { description: "The total number of files deleted." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + file: { + description: "The file that produced the error." + required: false + } + } } files_resumed_total: { description: "The total number of times Vector has resumed watching a file." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + file: { + description: "The file that produced the error." + required: false + } + } } files_unwatched_total: { description: "The total number of times Vector has stopped watching a file." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, file: {description: "The file that produced the error.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + file: { + description: "The file that produced the error." + required: false + } + } } grpc_server_handler_duration_seconds: { description: "The duration spent handling a gRPC request." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, grpc_method: {description: "The name of the method called on the gRPC service.", required: true}, grpc_service: {description: "The gRPC service name.", required: true}, grpc_status: {description: "The human-readable gRPC status code.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + grpc_method: { + description: "The name of the method called on the gRPC service." + required: true + } + grpc_service: { + description: "The gRPC service name." + required: true + } + grpc_status: { + description: "The human-readable gRPC status code." + required: true + } + } } grpc_server_messages_received_total: { description: "The total number of gRPC messages received." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, grpc_method: {description: "The name of the method called on the gRPC service.", required: true}, grpc_service: {description: "The gRPC service name.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + grpc_method: { + description: "The name of the method called on the gRPC service." + required: true + } + grpc_service: { + description: "The gRPC service name." + required: true + } + } } grpc_server_messages_sent_total: { description: "The total number of gRPC messages sent." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, grpc_method: {description: "The name of the method called on the gRPC service.", required: true}, grpc_service: {description: "The gRPC service name.", required: true}, grpc_status: {description: "The human-readable gRPC status code.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + grpc_method: { + description: "The name of the method called on the gRPC service." + required: true + } + grpc_service: { + description: "The gRPC service name." + required: true + } + grpc_status: { + description: "The human-readable gRPC status code." + required: true + } + } } http_client_error_rtt_seconds: { description: "The round-trip time (RTT) of HTTP requests that resulted in an error." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } http_client_errors_total: { description: "The total number of HTTP client errors encountered." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } http_client_requests_sent_total: { description: "The total number of sent HTTP requests, tagged with the request method." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, method: {description: "The HTTP method of the request.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + method: { + description: "The HTTP method of the request." + required: false + } + } } http_client_response_rtt_seconds: { description: "The round-trip time (RTT) of HTTP requests, tagged with the response code." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, status: {description: "The HTTP status code of the request.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + status: { + description: "The HTTP status code of the request." + required: false + } + } } http_client_responses_total: { description: "The total number of HTTP requests, tagged with the response code." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, status: {description: "The HTTP status code of the request.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + status: { + description: "The HTTP status code of the request." + required: false + } + } } http_client_rtt_seconds: { description: "The round-trip time (RTT) of HTTP requests." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } http_server_handler_duration_seconds: { description: "The duration spent handling an HTTP request." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, method: {description: "The HTTP method of the request.", required: false}, path: {description: "The path that produced the error.", required: true}, status: {description: "The HTTP status code of the request.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + method: { + description: "The HTTP method of the request." + required: false + } + path: { + description: "The path that produced the error." + required: true + } + status: { + description: "The HTTP status code of the request." + required: false + } + } } http_server_requests_received_total: { description: "The total number of HTTP requests received." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, method: {description: "The HTTP method of the request.", required: false}, path: {description: "The path that produced the error.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + method: { + description: "The HTTP method of the request." + required: false + } + path: { + description: "The path that produced the error." + required: true + } + } } http_server_responses_sent_total: { description: "The total number of HTTP responses sent." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, method: {description: "The HTTP method of the request.", required: false}, path: {description: "The path that produced the error.", required: true}, status: {description: "The HTTP status code of the request.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + method: { + description: "The HTTP method of the request." + required: false + } + path: { + description: "The path that produced the error." + required: true + } + status: { + description: "The HTTP status code of the request." + required: false + } + } } internal_metrics_cardinality: { description: "The total number of metrics emitted from the internal metrics registry." @@ -530,211 +3279,1275 @@ components: sources: internal_metrics: output: metrics: { description: "The total number of failures to parse a message as a JSON object." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } k8s_event_namespace_annotation_failures_total: { description: "The total number of failures to annotate Kubernetes events with namespace metadata." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } k8s_event_node_annotation_failures_total: { description: "The total number of failures to annotate Kubernetes events with node metadata." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } k8s_format_picker_edge_cases_total: { description: "The total number of edge cases encountered while picking format of the Kubernetes log message." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_consumed_messages_bytes_total: { description: "Total number of message bytes (including framing) received from Kafka brokers." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_consumed_messages_total: { description: "Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_consumer_lag: { description: "The Kafka consumer lag." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, topic_id: {description: "The Kafka topic id.", required: true}, partition_id: {description: "The Kafka partition id.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + topic_id: { + description: "The Kafka topic id." + required: true + } + partition_id: { + description: "The Kafka partition id." + required: true + } + } } kafka_produced_messages_bytes_total: { description: "Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_produced_messages_total: { description: "Total number of messages transmitted (produced) to Kafka brokers." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_queue_messages: { description: "Current number of messages in producer queues." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_queue_messages_bytes: { description: "Current total size of messages in producer queues." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_requests_bytes_total: { description: "Total number of bytes transmitted to Kafka brokers." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_requests_total: { description: "Total number of requests sent to Kafka brokers." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_responses_bytes_total: { description: "Total number of bytes received from Kafka brokers." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } kafka_responses_total: { description: "Total number of responses received from Kafka brokers." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } lua_memory_used_bytes: { description: "The total memory currently being used by the Lua runtime." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } memory_enrichment_table_byte_size: { description: "The total size in bytes of all objects stored in the in-memory enrichment table." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } memory_enrichment_table_failed_insertions: { description: "The total number of failed insertions into the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } memory_enrichment_table_failed_reads: { description: "The total number of failed reads from the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } memory_enrichment_table_flushes_total: { description: "The total number of flushes of the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } memory_enrichment_table_insertions_total: { description: "The total number of successful insertions into the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } memory_enrichment_table_objects_count: { description: "The number of objects currently stored in the in-memory enrichment table." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } memory_enrichment_table_reads_total: { description: "The total number of successful reads from the in-memory enrichment table." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } memory_enrichment_table_ttl_expirations: { description: "The total number of entries evicted from the in-memory enrichment table due to TTL expiration." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } metadata_refresh_failed_total: { description: "The total number of failed efforts to refresh AWS EC2 metadata." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } metadata_refresh_successful_total: { description: "The total number of AWS EC2 metadata refreshes." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } open_connections: { description: "The number of current open connections to Vector." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } open_files: { description: "The total number of open files." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } parse_errors_total: { description: "The total number of errors encountered while parsing." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } quit_total: { description: "The total number of times the Vector instance has quit." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } reloaded_total: { description: "The total number of times the Vector instance has been reloaded." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } rewritten_timestamp_events_total: { description: "The total number of events with rewrapped timestamps." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } s3_object_processing_failed_duration_seconds: { description: "The time taken to process an S3 object that failed, in seconds." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, bucket: {description: "The name of the S3 bucket.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + bucket: { + description: "The name of the S3 bucket." + required: true + } + } } s3_object_processing_succeeded_duration_seconds: { description: "The time taken to process an S3 object that succeeded, in seconds." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, bucket: {description: "The name of the S3 bucket.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + bucket: { + description: "The name of the S3 bucket." + required: true + } + } } source_buffer_max_byte_size: { description: "The maximum number of bytes the source buffer can hold." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } deprecated: true deprecated_message: "This metric has been deprecated in favor of `source_buffer_max_size_bytes`." } @@ -742,7 +4555,43 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum number of events the source buffer can hold." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } deprecated: true deprecated_message: "This metric has been deprecated in favor of `source_buffer_max_size_events`." } @@ -750,133 +4599,836 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum number of bytes the source buffer can hold." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } source_buffer_max_size_events: { description: "The maximum number of events the source buffer can hold." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } source_buffer_utilization: { description: "The utilization of the source buffer." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } source_buffer_utilization_level: { description: "The current utilization level of the source buffer." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } source_buffer_utilization_mean: { description: "The mean utilization of the source buffer, smoothed with an EWMA." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } source_lag_time_seconds: { description: "The difference between the timestamp recorded in each event and the time when it was ingested, expressed as fractional seconds." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } source_send_batch_latency_seconds: { description: "The time elapsed blocking on the downstream channel to accept an entire batch of events received at the source." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } source_send_latency_seconds: { description: "The time elapsed blocking on the downstream channel to accept a single chunk from a batch of events received at the source." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } splunk_pending_acks: { description: "The number of outstanding Splunk HEC indexer acknowledgement acks." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } sqs_message_defer_succeeded_total: { description: "The total number of successful deferrals of SQS messages." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } sqs_message_delete_succeeded_total: { description: "The total number of successful deletions of SQS messages." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } sqs_message_processing_succeeded_total: { description: "The total number of SQS messages successfully processed." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } sqs_message_receive_succeeded_total: { description: "The total number of times successfully receiving SQS messages." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } sqs_message_received_messages_total: { description: "The total number of received SQS messages." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } sqs_s3_event_record_ignored_total: { description: "The total number of times an S3 record in an SQS message was ignored." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, ignore_type: {description: "The reason for ignoring the S3 record", required: true, enum: {invalid_event_kind: "The kind of invalid event."}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + ignore_type: { + description: "The reason for ignoring the S3 record" + required: true + enum: invalid_event_kind: "The kind of invalid event." + } + } } stale_events_flushed_total: { description: "The number of stale events that Vector has flushed." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } started_total: { description: "The total number of times the Vector instance has been started." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } stopped_total: { description: "The total number of times the Vector instance has been stopped." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } tag_cardinality_tracked_keys: { description: "The number of tag keys currently being tracked by the tag cardinality limit transform." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } tag_cardinality_untracked_events_total: { description: "The total number of events that contained a tag which exceeded the configured cardinality limit." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } tag_value_limit_exceeded_total: { description: "The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, metric_name: {description: "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", required: false}, tag_key: {description: "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + metric_name: { + description: "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled." + required: false + } + tag_key: { + description: "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled." + required: false + } + } } transform_buffer_max_byte_size: { description: "The maximum number of bytes the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } deprecated: true deprecated_message: "This metric has been deprecated in favor of `transform_buffer_max_size_bytes`." } @@ -884,7 +5436,43 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum number of events the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } deprecated: true deprecated_message: "This metric has been deprecated in favor of `transform_buffer_max_size_events`." } @@ -892,96 +5480,519 @@ components: sources: internal_metrics: output: metrics: { description: "The maximum number of bytes the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } transform_buffer_max_size_events: { description: "The maximum number of events the buffer that feeds into a transform can hold." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } transform_buffer_utilization: { description: "The utilization of the buffer that feeds into a transform." type: "histogram" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } transform_buffer_utilization_level: { description: "The current utilization level of the buffer that feeds into a transform." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } transform_buffer_utilization_mean: { description: "The mean utilization of the buffer that feeds into a transform, smoothed with an EWMA." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, output: {description: "The specific output of the component.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + output: { + description: "The specific output of the component." + required: false + } + } } uptime_seconds: { description: "The number of seconds the Vector instance has been running." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } utf8_convert_errors_total: { description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}, mode: {description: "The connection mode used by the component.", required: true, enum: {udp: "User Datagram Protocol"}}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + mode: { + description: "The connection mode used by the component." + required: true + enum: udp: "User Datagram Protocol" + } + } } utilization: { description: "The current utilization of this component, expressed as a value from 0 to 1." type: "gauge" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } value_limit_reached_total: { description: "The total number of times the value limit was reached." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } websocket_bytes_sent_total: { description: "The total number of bytes sent over WebSocket connections." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } websocket_messages_sent_total: { description: "The total number of messages sent over WebSocket connections." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}, component_kind: {description: "The Vector component kind.", required: true, enum: {sink: "Vector sink components", source: "Vector source components", transform: "Vector transform components"}}, component_id: {description: "The Vector component ID.", required: true}, component_type: {description: "The Vector component type.", required: true}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + component_kind: { + description: "The Vector component kind." + required: true + enum: { + sink: "Vector sink components" + source: "Vector source components" + transform: "Vector transform components" + } + } + component_id: { + description: "The Vector component ID." + required: true + examples: ["my_source", "my_sink"] + } + component_type: { + description: "The Vector component type." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + } } windows_service_install_total: { description: "The total number of times the Windows service has been installed." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } windows_service_restart_total: { description: "The total number of times the Windows service has been restarted." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } windows_service_start_total: { description: "The total number of times the Windows service has been started." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } windows_service_stop_total: { description: "The total number of times the Windows service has been stopped." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } windows_service_uninstall_total: { description: "The total number of times the Windows service has been uninstalled." type: "counter" default_namespace: "vector" - tags: {pid: {description: "The process ID of the Vector instance.", required: false}, host: {description: "The hostname of the system Vector is running on.", required: false}} + tags: { + pid: { + description: "The process ID of the Vector instance." + required: false + examples: [ + "4232", + ] + } + host: { + description: "The hostname of the system Vector is running on." + required: false + examples: ["my-host.local"] + } + } } } From 135d91edb8c3ab5e3c159deaee9d4b0d747efa91 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 16:15:51 -0400 Subject: [PATCH 37/43] Fix description generation and wrongly updated descriptions --- .../src/internal_event/metric_name.rs | 51 ++++++++------- .../src/internal_event/metric_tags.rs | 2 +- .../commands/build/component_docs/runner.rs | 13 +++- .../internal_metric_descriptions.cue | 65 ++++++++++++------- 4 files changed, 80 insertions(+), 51 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index aab1b747145ae..8cb64d49c3674 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -315,14 +315,19 @@ pub enum CounterName { #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] TagCardinalityUntrackedEventsTotal, - /// The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. + /// The total number of events discarded because the tag has been rejected after + /// hitting the configured `value_limit`. When `internal_metrics.include_extended_tags` + /// is enabled in the `tag_cardinality_limit` transform, this metric includes + /// `metric_name` and `tag_key` labels. By default, this metric has no labels to + /// keep cardinality low. #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "metric_name": {"description": "The name of the metric whose tag value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false}, "tag_key": {"description": "The key of the tag whose value limit was exceeded. Only present when `internal_metrics.include_extended_tags` is enabled.", "required": false} }))))] TagValueLimitExceededTotal, - /// The total number of times the value limit was reached. + /// The total number of times new values for a key have been rejected because the + /// value limit has been reached. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] ValueLimitReachedTotal, @@ -370,7 +375,7 @@ pub enum CounterName { #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] K8sDockerFormatParseFailuresTotal, - /// The total number of times an S3 record in an SQS message was ignored. + /// The total number of times an S3 record in an SQS message was ignored (for an event that was not `ObjectCreated`). #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ "ignore_type": { "description": "The reason for ignoring the S3 record", @@ -441,8 +446,8 @@ pub enum CounterName { pub enum HistogramName { /// A histogram of the number of events passed in each internal batch in Vector's internal topology. /// - /// Note that this is separate than sink-level batching. It is mostly useful for low level - /// debugging performance issues in Vector due to small internal batches. + /// Note that this is separate than sink-level batching. It is mostly useful for low level debugging + /// performance issues in Vector due to small internal batches. #[configurable(metadata(docs::tags = &*COMPONENT_RECEIVED_EVENTS_TAGS))] ComponentReceivedEventsCount, @@ -537,11 +542,11 @@ pub enum HistogramName { #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] HttpClientErrorRttSeconds, - /// The utilization of the source buffer. + /// The utilization level of the source buffer. The outputs of the source send data to this buffer. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferUtilization, - /// The utilization of the buffer that feeds into a transform. + /// The utilization level of the buffer that feeds into a transform. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferUtilization, } @@ -597,46 +602,46 @@ pub enum GaugeName { #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] ComponentLatencyMeanSeconds, - /// The maximum number of events the source buffer can hold. + /// The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer. #[configurable( - deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_events`." + deprecated = "This metric has been deprecated in favor of [`source_buffer_max_size_events`](#source_buffer_max_size_events)." )] #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferMaxEventSize, - /// The maximum number of bytes the source buffer can hold. + /// The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer. #[configurable( - deprecated = "This metric has been deprecated in favor of `source_buffer_max_size_bytes`." + deprecated = "This metric has been deprecated in favor of [`source_buffer_max_size_bytes`](#source_buffer_max_size_bytes)." )] #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferMaxByteSize, - /// The maximum number of events the source buffer can hold. + /// The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferMaxSizeEvents, - /// The maximum number of bytes the source buffer can hold. + /// The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferMaxSizeBytes, - /// The current utilization level of the source buffer. + /// The current utilization level of the source buffer. The outputs of the source send data to this buffer. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferUtilizationLevel, - /// The mean utilization of the source buffer, smoothed with an EWMA. + /// The mean utilization level of the source buffer. The outputs of the source send data to this buffer. The mean utilization is smoothed over time using an exponentially weighted moving average (EWMA). #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] SourceBufferUtilizationMean, /// The maximum number of events the buffer that feeds into a transform can hold. #[configurable( - deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_events`." + deprecated = "This metric has been deprecated in favor of [`transform_buffer_max_size_events`](#transform_buffer_max_size_events)." )] #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferMaxEventSize, /// The maximum number of bytes the buffer that feeds into a transform can hold. #[configurable( - deprecated = "This metric has been deprecated in favor of `transform_buffer_max_size_bytes`." + deprecated = "This metric has been deprecated in favor of [`transform_buffer_max_size_bytes`](#transform_buffer_max_size_bytes)." )] #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferMaxByteSize, @@ -653,7 +658,7 @@ pub enum GaugeName { #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferUtilizationLevel, - /// The mean utilization of the buffer that feeds into a transform, smoothed with an EWMA. + /// The mean utilization level of the buffer that feeds into a transform. This value is smoothed over time using an exponentially weighted moving average (EWMA). #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_OUTPUT))] TransformBufferUtilizationMean, @@ -675,7 +680,7 @@ pub enum GaugeName { /// The number of events currently in the buffer. #[configurable( - deprecated = "This metric has been deprecated in favor of `buffer_size_events`." + deprecated = "This metric has been deprecated in favor of [`buffer_size_events`](#buffer_size_events)." )] #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferEvents, @@ -689,11 +694,11 @@ pub enum GaugeName { BufferSizeBytes, /// The number of bytes currently in the buffer. - #[configurable(deprecated = "This metric has been deprecated in favor of `buffer_size_bytes`.")] + #[configurable(deprecated = "This metric has been deprecated in favor of [`buffer_size_bytes`](#buffer_size_bytes).")] #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferByteSize, - /// The current utilization of this component, expressed as a value from 0 to 1. + /// A ratio from 0 to 1 of the load on a component. A value of 0 would indicate a completely idle component that is simply waiting for input. A value of 1 would indicate a that is never idle. This value is updated every 5 seconds. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] Utilization, @@ -705,11 +710,11 @@ pub enum GaugeName { #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] OpenFiles, - /// The number of seconds the Vector instance has been running. + /// The total number of seconds the Vector instance has been up. #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] UptimeSeconds, - /// Pseudo-metric that provides build information for the Vector instance. + /// Has a fixed value of 1.0. Contains build information such as Rust and Vector versions. #[configurable(metadata(docs::tags = merge_lazy(&INTERNAL_METRICS_TAGS, json!({ "debug": {"description": "Whether this is a debug build of Vector", "required": true}, "version": {"description": "Vector version.", "required": true}, diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index 8a1de70f5caa1..dd7c56c8a3786 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -68,7 +68,7 @@ pub static COMPONENT_TAGS_GRPC_ALL: LazyLock = LazyLock::new(|| { merge_lazy( &COMPONENT_TAGS_GRPC_METHOD_SERVICE, json!({ - "grpc_status": {"description": "The human-readable gRPC status code.", "required": true} + "grpc_status": {"description": "The human-readable [gRPC status code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html).", "required": true} }), ) }); diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index 7e0977cec10a2..c55861d6e8dbb 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -381,8 +381,17 @@ fn collect_metric_entries( let Some(name) = variant.get("const").and_then(Value::as_str) else { continue; }; - let Some(desc) = variant.get("description").and_then(Value::as_str) else { - continue; + let title = variant.get("title").and_then(Value::as_str); + let body = variant.get("description").and_then(Value::as_str); + let desc_owned; + let desc = match (title, body) { + (Some(t), Some(b)) => { + desc_owned = format!("{t}\n\n{b}"); + desc_owned.as_str() + } + (Some(t), None) => t, + (None, Some(b)) => b, + (None, None) => continue, }; let tags = match variant.get("_metadata").and_then(|m| m.get("docs::tags")) { Some(v @ Value::Object(_)) => v.clone(), diff --git a/website/cue/reference/generated/internal_metric_descriptions.cue b/website/cue/reference/generated/internal_metric_descriptions.cue index 98b6809d844f7..25c6734f6058e 100644 --- a/website/cue/reference/generated/internal_metric_descriptions.cue +++ b/website/cue/reference/generated/internal_metric_descriptions.cue @@ -519,7 +519,7 @@ components: sources: internal_metrics: output: metrics: { } } deprecated: true - deprecated_message: "This metric has been deprecated in favor of `buffer_size_bytes`." + deprecated_message: "This metric has been deprecated in favor of [`buffer_size_bytes`](#buffer_size_bytes)." } buffer_discarded_bytes_total: { description: "The number of bytes dropped by this non-blocking buffer." @@ -673,7 +673,7 @@ components: sources: internal_metrics: output: metrics: { } } deprecated: true - deprecated_message: "This metric has been deprecated in favor of `buffer_size_events`." + deprecated_message: "This metric has been deprecated in favor of [`buffer_size_events`](#buffer_size_events)." } buffer_max_byte_size: { description: "The maximum size in bytes that the buffer can store." @@ -1094,7 +1094,7 @@ components: sources: internal_metrics: output: metrics: { } } build_info: { - description: "Pseudo-metric that provides build information for the Vector instance." + description: "Has a fixed value of 1.0. Contains build information such as Rust and Vector versions." type: "gauge" default_namespace: "vector" tags: { @@ -1522,6 +1522,8 @@ components: sources: internal_metrics: output: metrics: { } component_latency_mean_seconds: { description: """ + The mean elapsed time, in fractional seconds, that an event spends in a single transform. + This includes both the time spent queued in the transform's input buffer and the time spent executing the transform itself. This value is smoothed over time using an exponentially weighted moving average (EWMA). @@ -1545,6 +1547,8 @@ components: sources: internal_metrics: output: metrics: { } component_latency_seconds: { description: """ + The elapsed time, in fractional seconds, that an event spends in a single transform. + This includes both the time spent queued in the transform's input buffer and the time spent executing the transform itself. """ @@ -1783,8 +1787,10 @@ components: sources: internal_metrics: output: metrics: { } component_received_events_count: { description: """ - Note that this is separate than sink-level batching. It is mostly useful for low level - debugging performance issues in Vector due to small internal batches. + A histogram of the number of events passed in each internal batch in Vector's internal topology. + + Note that this is separate than sink-level batching. It is mostly useful for low level debugging + performance issues in Vector due to small internal batches. """ type: "histogram" default_namespace: "vector" @@ -2776,7 +2782,7 @@ components: sources: internal_metrics: output: metrics: { required: true } grpc_status: { - description: "The human-readable gRPC status code." + description: "The human-readable [gRPC status code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html)." required: true } } @@ -2872,7 +2878,7 @@ components: sources: internal_metrics: output: metrics: { required: true } grpc_status: { - description: "The human-readable gRPC status code." + description: "The human-readable [gRPC status code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html)." required: true } } @@ -4508,7 +4514,7 @@ components: sources: internal_metrics: output: metrics: { } } source_buffer_max_byte_size: { - description: "The maximum number of bytes the source buffer can hold." + description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: { @@ -4549,10 +4555,10 @@ components: sources: internal_metrics: output: metrics: { } } deprecated: true - deprecated_message: "This metric has been deprecated in favor of `source_buffer_max_size_bytes`." + deprecated_message: "This metric has been deprecated in favor of [`source_buffer_max_size_bytes`](#source_buffer_max_size_bytes)." } source_buffer_max_event_size: { - description: "The maximum number of events the source buffer can hold." + description: "The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: { @@ -4593,10 +4599,10 @@ components: sources: internal_metrics: output: metrics: { } } deprecated: true - deprecated_message: "This metric has been deprecated in favor of `source_buffer_max_size_events`." + deprecated_message: "This metric has been deprecated in favor of [`source_buffer_max_size_events`](#source_buffer_max_size_events)." } source_buffer_max_size_bytes: { - description: "The maximum number of bytes the source buffer can hold." + description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: { @@ -4638,7 +4644,7 @@ components: sources: internal_metrics: output: metrics: { } } source_buffer_max_size_events: { - description: "The maximum number of events the source buffer can hold." + description: "The maximum number of events the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: { @@ -4680,7 +4686,7 @@ components: sources: internal_metrics: output: metrics: { } } source_buffer_utilization: { - description: "The utilization of the source buffer." + description: "The utilization level of the source buffer. The outputs of the source send data to this buffer." type: "histogram" default_namespace: "vector" tags: { @@ -4722,7 +4728,7 @@ components: sources: internal_metrics: output: metrics: { } } source_buffer_utilization_level: { - description: "The current utilization level of the source buffer." + description: "The current utilization level of the source buffer. The outputs of the source send data to this buffer." type: "gauge" default_namespace: "vector" tags: { @@ -4764,7 +4770,7 @@ components: sources: internal_metrics: output: metrics: { } } source_buffer_utilization_mean: { - description: "The mean utilization of the source buffer, smoothed with an EWMA." + description: "The mean utilization level of the source buffer. The outputs of the source send data to this buffer. The mean utilization is smoothed over time using an exponentially weighted moving average (EWMA)." type: "gauge" default_namespace: "vector" tags: { @@ -5148,7 +5154,7 @@ components: sources: internal_metrics: output: metrics: { } } sqs_s3_event_record_ignored_total: { - description: "The total number of times an S3 record in an SQS message was ignored." + description: "The total number of times an S3 record in an SQS message was ignored (for an event that was not `ObjectCreated`)." type: "counter" default_namespace: "vector" tags: { @@ -5343,7 +5349,13 @@ components: sources: internal_metrics: output: metrics: { } } tag_value_limit_exceeded_total: { - description: "The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`." + description: """ + The total number of events discarded because the tag has been rejected after + hitting the configured `value_limit`. When `internal_metrics.include_extended_tags` + is enabled in the `tag_cardinality_limit` transform, this metric includes + `metric_name` and `tag_key` labels. By default, this metric has no labels to + keep cardinality low. + """ type: "counter" default_namespace: "vector" tags: { @@ -5430,7 +5442,7 @@ components: sources: internal_metrics: output: metrics: { } } deprecated: true - deprecated_message: "This metric has been deprecated in favor of `transform_buffer_max_size_bytes`." + deprecated_message: "This metric has been deprecated in favor of [`transform_buffer_max_size_bytes`](#transform_buffer_max_size_bytes)." } transform_buffer_max_event_size: { description: "The maximum number of events the buffer that feeds into a transform can hold." @@ -5474,7 +5486,7 @@ components: sources: internal_metrics: output: metrics: { } } deprecated: true - deprecated_message: "This metric has been deprecated in favor of `transform_buffer_max_size_events`." + deprecated_message: "This metric has been deprecated in favor of [`transform_buffer_max_size_events`](#transform_buffer_max_size_events)." } transform_buffer_max_size_bytes: { description: "The maximum number of bytes the buffer that feeds into a transform can hold." @@ -5561,7 +5573,7 @@ components: sources: internal_metrics: output: metrics: { } } transform_buffer_utilization: { - description: "The utilization of the buffer that feeds into a transform." + description: "The utilization level of the buffer that feeds into a transform." type: "histogram" default_namespace: "vector" tags: { @@ -5645,7 +5657,7 @@ components: sources: internal_metrics: output: metrics: { } } transform_buffer_utilization_mean: { - description: "The mean utilization of the buffer that feeds into a transform, smoothed with an EWMA." + description: "The mean utilization level of the buffer that feeds into a transform. This value is smoothed over time using an exponentially weighted moving average (EWMA)." type: "gauge" default_namespace: "vector" tags: { @@ -5687,7 +5699,7 @@ components: sources: internal_metrics: output: metrics: { } } uptime_seconds: { - description: "The number of seconds the Vector instance has been running." + description: "The total number of seconds the Vector instance has been up." type: "gauge" default_namespace: "vector" tags: { @@ -5749,7 +5761,7 @@ components: sources: internal_metrics: output: metrics: { } } utilization: { - description: "The current utilization of this component, expressed as a value from 0 to 1." + description: "A ratio from 0 to 1 of the load on a component. A value of 0 would indicate a completely idle component that is simply waiting for input. A value of 1 would indicate a that is never idle. This value is updated every 5 seconds." type: "gauge" default_namespace: "vector" tags: { @@ -5787,7 +5799,10 @@ components: sources: internal_metrics: output: metrics: { } } value_limit_reached_total: { - description: "The total number of times the value limit was reached." + description: """ + The total number of times new values for a key have been rejected because the + value limit has been reached. + """ type: "counter" default_namespace: "vector" tags: { From 47cd9cba3273900b44e7482453b68f5e3f046e86 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 16:48:13 -0400 Subject: [PATCH 38/43] Add deprecation notice to buffer-max-byte-size/buffer-max-event-size --- .../src/internal_event/metric_name.rs | 14 +++++++++++--- .../generated/internal_metric_descriptions.cue | 4 ++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index 8cb64d49c3674..d857166d86eb1 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -667,7 +667,10 @@ pub enum GaugeName { BufferMaxSizeEvents, /// The maximum size in events that the buffer can store. - #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] + #[configurable( + deprecated = "This metric has been deprecated in favor of [`buffer_max_size_events`](#buffer_max_size_events).", + metadata(docs::tags = &*COMPONENT_TAGS) + )] BufferMaxEventSize, /// The maximum number of bytes in the buffer. @@ -675,7 +678,10 @@ pub enum GaugeName { BufferMaxSizeBytes, /// The maximum size in bytes that the buffer can store. - #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] + #[configurable( + deprecated = "This metric has been deprecated in favor of [`buffer_max_size_bytes`](#buffer_max_size_bytes).", + metadata(docs::tags = &*COMPONENT_TAGS) + )] BufferMaxByteSize, /// The number of events currently in the buffer. @@ -694,7 +700,9 @@ pub enum GaugeName { BufferSizeBytes, /// The number of bytes currently in the buffer. - #[configurable(deprecated = "This metric has been deprecated in favor of [`buffer_size_bytes`](#buffer_size_bytes).")] + #[configurable( + deprecated = "This metric has been deprecated in favor of [`buffer_size_bytes`](#buffer_size_bytes)." + )] #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferByteSize, diff --git a/website/cue/reference/generated/internal_metric_descriptions.cue b/website/cue/reference/generated/internal_metric_descriptions.cue index 25c6734f6058e..dce20ed376209 100644 --- a/website/cue/reference/generated/internal_metric_descriptions.cue +++ b/website/cue/reference/generated/internal_metric_descriptions.cue @@ -712,6 +712,8 @@ components: sources: internal_metrics: output: metrics: { examples: ["file", "http", "honeycomb", "splunk_hec"] } } + deprecated: true + deprecated_message: "This metric has been deprecated in favor of [`buffer_max_size_bytes`](#buffer_max_size_bytes)." } buffer_max_event_size: { description: "The maximum size in events that the buffer can store." @@ -750,6 +752,8 @@ components: sources: internal_metrics: output: metrics: { examples: ["file", "http", "honeycomb", "splunk_hec"] } } + deprecated: true + deprecated_message: "This metric has been deprecated in favor of [`buffer_max_size_events`](#buffer_max_size_events)." } buffer_max_size_bytes: { description: "The maximum number of bytes in the buffer." From 71bc4322a61df71165bce0911e4700d2db914fe0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 16:49:35 -0400 Subject: [PATCH 39/43] fmt --- vdev/src/commands/build/component_docs/runner.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/vdev/src/commands/build/component_docs/runner.rs b/vdev/src/commands/build/component_docs/runner.rs index c55861d6e8dbb..03f7fa44804ac 100644 --- a/vdev/src/commands/build/component_docs/runner.rs +++ b/vdev/src/commands/build/component_docs/runner.rs @@ -362,9 +362,7 @@ fn render_and_import_generated_top_level_config_schema( } /// Collects all metric variants from `metric_schemas` into an alphabetically-sorted JSON map. -fn collect_metric_entries( - metric_schemas: &Value, -) -> std::collections::BTreeMap { +fn collect_metric_entries(metric_schemas: &Value) -> std::collections::BTreeMap { let mut metrics = std::collections::BTreeMap::new(); for (metric_type, schema) in [ ("counter", metric_schemas.get("counters")), @@ -455,8 +453,7 @@ fn generate_internal_metric_descriptions( }); let json_str = serde_json::to_string_pretty(&data)?; - let json_file = - write_to_temp_file("internal-metric-descriptions-", ".json", &json_str)?; + let json_file = write_to_temp_file("internal-metric-descriptions-", ".json", &json_str)?; if let Some(parent) = out_path.parent() { fs::create_dir_all(parent)?; From f64f3e857f5f6bb3011b616de24ea17f754beca2 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 17:21:51 -0400 Subject: [PATCH 40/43] fix(metrics): correct descriptions and add missing tags/metrics to sink telemetry --- .../src/internal_event/metric_name.rs | 18 +++++++++--------- .../src/internal_event/metric_tags.rs | 9 +++++++++ website/cue/reference/components/sinks.cue | 2 ++ .../generated/internal_metric_descriptions.cue | 14 +++++++++++--- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index d857166d86eb1..e91b89185d5d9 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -5,10 +5,10 @@ use vector_config::configurable_component; use super::metric_tags::{ COMPONENT_RECEIVED_EVENTS_TAGS, COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS, COMPONENT_TAGS, COMPONENT_TAGS_ERROR_TYPE_STAGE, COMPONENT_TAGS_GRPC_ALL, COMPONENT_TAGS_GRPC_METHOD_SERVICE, - COMPONENT_TAGS_HTTP_ALL, COMPONENT_TAGS_HTTP_METHOD, COMPONENT_TAGS_HTTP_METHOD_PATH, - COMPONENT_TAGS_HTTP_STATUS, COMPONENT_TAGS_OUTPUT, INTERNAL_METRICS_TAGS, - INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, S3_OBJECT_PROCESSING_TAGS, - merge_lazy, + COMPONENT_TAGS_HTTP_ALL, COMPONENT_TAGS_HTTP_ERROR_KIND, COMPONENT_TAGS_HTTP_METHOD, + COMPONENT_TAGS_HTTP_METHOD_PATH, COMPONENT_TAGS_HTTP_STATUS, COMPONENT_TAGS_OUTPUT, + INTERNAL_METRICS_TAGS, INTERNAL_METRICS_TAGS_FILE, INTERNAL_METRICS_TAGS_REASON, + S3_OBJECT_PROCESSING_TAGS, merge_lazy, }; /// Canonical list of all per-component internal counter metric names emitted by Vector. @@ -204,7 +204,7 @@ pub enum CounterName { GrpcServerMessagesSentTotal, /// The total number of HTTP client errors encountered. - #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_HTTP_ERROR_KIND))] HttpClientErrorsTotal, /// The total number of sent HTTP requests, tagged with the request method. @@ -275,7 +275,7 @@ pub enum CounterName { #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] ReloadedTotal, - /// The total number of events with rewrapped timestamps. + /// The total number of events whose timestamps were rewritten to maintain ordering. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] RewrittenTimestampEventsTotal, @@ -311,7 +311,7 @@ pub enum CounterName { #[configurable(metadata(docs::tags = &*INTERNAL_METRICS_TAGS))] StoppedTotal, - /// The total number of events that contained a tag which exceeded the configured cardinality limit. + /// The total number of events whose tag keys are no longer tracked because `max_tracked_keys` was reached. These events pass through the transform unchecked. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] TagCardinalityUntrackedEventsTotal, @@ -502,7 +502,7 @@ pub enum HistogramName { #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyPastRttMean, - /// The number of times the concurrency limit was reached. + /// A boolean sample (1.0 if the concurrency limit was reached during the most recent limit update, 0.0 otherwise). #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyReachedLimit, @@ -539,7 +539,7 @@ pub enum HistogramName { HttpClientResponseRttSeconds, /// The round-trip time (RTT) of HTTP requests that resulted in an error. - #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] + #[configurable(metadata(docs::tags = &*COMPONENT_TAGS_HTTP_ERROR_KIND))] HttpClientErrorRttSeconds, /// The utilization level of the source buffer. The outputs of the source send data to this buffer. diff --git a/lib/vector-common/src/internal_event/metric_tags.rs b/lib/vector-common/src/internal_event/metric_tags.rs index dd7c56c8a3786..b368ab7d449f2 100644 --- a/lib/vector-common/src/internal_event/metric_tags.rs +++ b/lib/vector-common/src/internal_event/metric_tags.rs @@ -211,6 +211,15 @@ pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock = LazyLock::new pub static COMPONENT_RECEIVED_EVENTS_TAGS: LazyLock = LazyLock::new(|| COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS.clone()); +pub static COMPONENT_TAGS_HTTP_ERROR_KIND: LazyLock = LazyLock::new(|| { + merge_lazy( + &COMPONENT_TAGS, + json!({ + "error_kind": {"description": "The kind of HTTP error encountered (e.g. connection refused, connection reset).", "required": true} + }), + ) +}); + pub static S3_OBJECT_PROCESSING_TAGS: LazyLock = LazyLock::new(|| { merge_lazy( &COMPONENT_TAGS, diff --git a/website/cue/reference/components/sinks.cue b/website/cue/reference/components/sinks.cue index a3c9c9b9be9b2..8e245ef5cb0e6 100644 --- a/website/cue/reference/components/sinks.cue +++ b/website/cue/reference/components/sinks.cue @@ -666,7 +666,9 @@ components: sinks: [Name=string]: { buffer_size_bytes: components.sources.internal_metrics.output.metrics.buffer_size_bytes buffer_size_events: components.sources.internal_metrics.output.metrics.buffer_size_events buffer_events: components.sources.internal_metrics.output.metrics.buffer_events + buffer_received_bytes_total: components.sources.internal_metrics.output.metrics.buffer_received_bytes_total buffer_received_events_total: components.sources.internal_metrics.output.metrics.buffer_received_events_total + buffer_sent_bytes_total: components.sources.internal_metrics.output.metrics.buffer_sent_bytes_total buffer_sent_events_total: components.sources.internal_metrics.output.metrics.buffer_sent_events_total component_discarded_events_total: components.sources.internal_metrics.output.metrics.component_discarded_events_total component_errors_total: components.sources.internal_metrics.output.metrics.component_errors_total diff --git a/website/cue/reference/generated/internal_metric_descriptions.cue b/website/cue/reference/generated/internal_metric_descriptions.cue index dce20ed376209..4eb6ae36f5e69 100644 --- a/website/cue/reference/generated/internal_metric_descriptions.cue +++ b/website/cue/reference/generated/internal_metric_descriptions.cue @@ -311,7 +311,7 @@ components: sources: internal_metrics: output: metrics: { } } adaptive_concurrency_reached_limit: { - description: "The number of times the concurrency limit was reached." + description: "A boolean sample (1.0 if the concurrency limit was reached during the most recent limit update, 0.0 otherwise)." type: "histogram" default_namespace: "vector" tags: { @@ -2923,6 +2923,10 @@ components: sources: internal_metrics: output: metrics: { required: true examples: ["file", "http", "honeycomb", "splunk_hec"] } + error_kind: { + description: "The kind of HTTP error encountered (e.g. connection refused, connection reset)." + required: true + } } } http_client_errors_total: { @@ -2961,6 +2965,10 @@ components: sources: internal_metrics: output: metrics: { required: true examples: ["file", "http", "honeycomb", "splunk_hec"] } + error_kind: { + description: "The kind of HTTP error encountered (e.g. connection refused, connection reset)." + required: true + } } } http_client_requests_sent_total: { @@ -4396,7 +4404,7 @@ components: sources: internal_metrics: output: metrics: { } } rewritten_timestamp_events_total: { - description: "The total number of events with rewrapped timestamps." + description: "The total number of events whose timestamps were rewritten to maintain ordering." type: "counter" default_namespace: "vector" tags: { @@ -5315,7 +5323,7 @@ components: sources: internal_metrics: output: metrics: { } } tag_cardinality_untracked_events_total: { - description: "The total number of events that contained a tag which exceeded the configured cardinality limit." + description: "The total number of events whose tag keys are no longer tracked because `max_tracked_keys` was reached. These events pass through the transform unchecked." type: "counter" default_namespace: "vector" tags: { From d143e7a1014b38d56720e7bf94cecac20112c954 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 17:38:25 -0400 Subject: [PATCH 41/43] fix(metrics): correct inaccurate metric descriptions --- lib/vector-common/src/internal_event/metric_name.rs | 10 +++++----- .../generated/internal_metric_descriptions.cue | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index e91b89185d5d9..cd9772a726039 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -82,11 +82,11 @@ pub enum CounterName { #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferSentBytesTotal, - /// The number of events dropped by this non-blocking buffer. + /// The number of events dropped by this buffer. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferDiscardedEventsTotal, - /// The number of bytes dropped by this non-blocking buffer. + /// The number of bytes dropped by this buffer. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] BufferDiscardedBytesTotal, @@ -419,7 +419,7 @@ pub enum CounterName { /// The total number of errors reading datagram. #[configurable(metadata(docs::tags = merge_lazy(&COMPONENT_TAGS, json!({ - "mode": {"description": "", "required": true, "enum": {"udp": "User Datagram Protocol"}} + "mode": {"description": "The connection mode used by the component.", "required": true, "enum": {"udp": "User Datagram Protocol"}} }))))] ConnectionReadErrorsTotal, @@ -482,7 +482,7 @@ pub enum HistogramName { #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyAveragedRtt, - /// The amount of back pressure on the current component. + /// A boolean sample (1.0 if back pressure was observed during the current window, 0.0 otherwise). #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyBackPressure, @@ -498,7 +498,7 @@ pub enum HistogramName { #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyObservedRtt, - /// The mean round-trip time (RTT) for the current window. + /// The mean round-trip time (RTT) from past windows, used as the baseline for adaptive concurrency decisions. #[configurable(metadata(docs::tags = &*COMPONENT_TAGS))] AdaptiveConcurrencyPastRttMean, diff --git a/website/cue/reference/generated/internal_metric_descriptions.cue b/website/cue/reference/generated/internal_metric_descriptions.cue index 4eb6ae36f5e69..7cd8650f9eeee 100644 --- a/website/cue/reference/generated/internal_metric_descriptions.cue +++ b/website/cue/reference/generated/internal_metric_descriptions.cue @@ -121,7 +121,7 @@ components: sources: internal_metrics: output: metrics: { } } adaptive_concurrency_back_pressure: { - description: "The amount of back pressure on the current component." + description: "A boolean sample (1.0 if back pressure was observed during the current window, 0.0 otherwise)." type: "histogram" default_namespace: "vector" tags: { @@ -273,7 +273,7 @@ components: sources: internal_metrics: output: metrics: { } } adaptive_concurrency_past_rtt_mean: { - description: "The mean round-trip time (RTT) for the current window." + description: "The mean round-trip time (RTT) from past windows, used as the baseline for adaptive concurrency decisions." type: "histogram" default_namespace: "vector" tags: { @@ -522,7 +522,7 @@ components: sources: internal_metrics: output: metrics: { deprecated_message: "This metric has been deprecated in favor of [`buffer_size_bytes`](#buffer_size_bytes)." } buffer_discarded_bytes_total: { - description: "The number of bytes dropped by this non-blocking buffer." + description: "The number of bytes dropped by this buffer." type: "counter" default_namespace: "vector" tags: { @@ -560,7 +560,7 @@ components: sources: internal_metrics: output: metrics: { } } buffer_discarded_events_total: { - description: "The number of events dropped by this non-blocking buffer." + description: "The number of events dropped by this buffer." type: "counter" default_namespace: "vector" tags: { @@ -2236,7 +2236,7 @@ components: sources: internal_metrics: output: metrics: { examples: ["file", "http", "honeycomb", "splunk_hec"] } mode: { - description: "" + description: "The connection mode used by the component." required: true enum: udp: "User Datagram Protocol" } From e140e7fdc6449f4381a648f69ec106cb4e1ecf42 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 18:04:42 -0400 Subject: [PATCH 42/43] Refactor match into kind-only/kind + limit matches --- .../src/topology/channel/limited_queue.rs | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index e341c76446cbe..e2c2eb12fd405 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -144,41 +144,42 @@ impl Metrics { let ewma_half_life_seconds = ewma_half_life_seconds.unwrap_or(DEFAULT_EWMA_HALF_LIFE_SECONDS); let ChannelMetricMetadata { kind, output } = metadata; - let (max_name, legacy_name, histogram_name, level_name, mean_name, max_value) = - match (kind, limit) { - (BufferChannelKind::Source, MemoryBufferSize::MaxEvents(n)) => ( - GaugeName::SourceBufferMaxSizeEvents, - GaugeName::SourceBufferMaxEventSize, - HistogramName::SourceBufferUtilization, - GaugeName::SourceBufferUtilizationLevel, - GaugeName::SourceBufferUtilizationMean, - n.get() as f64, - ), - (BufferChannelKind::Source, MemoryBufferSize::MaxSize(n)) => ( - GaugeName::SourceBufferMaxSizeBytes, - GaugeName::SourceBufferMaxByteSize, - HistogramName::SourceBufferUtilization, - GaugeName::SourceBufferUtilizationLevel, - GaugeName::SourceBufferUtilizationMean, - n.get() as f64, - ), - (BufferChannelKind::Transform, MemoryBufferSize::MaxEvents(n)) => ( - GaugeName::TransformBufferMaxSizeEvents, - GaugeName::TransformBufferMaxEventSize, - HistogramName::TransformBufferUtilization, - GaugeName::TransformBufferUtilizationLevel, - GaugeName::TransformBufferUtilizationMean, - n.get() as f64, - ), - (BufferChannelKind::Transform, MemoryBufferSize::MaxSize(n)) => ( - GaugeName::TransformBufferMaxSizeBytes, - GaugeName::TransformBufferMaxByteSize, - HistogramName::TransformBufferUtilization, - GaugeName::TransformBufferUtilizationLevel, - GaugeName::TransformBufferUtilizationMean, - n.get() as f64, - ), - }; + + let (histogram_name, level_name, mean_name) = match kind { + BufferChannelKind::Source => ( + HistogramName::SourceBufferUtilization, + GaugeName::SourceBufferUtilizationLevel, + GaugeName::SourceBufferUtilizationMean, + ), + BufferChannelKind::Transform => ( + HistogramName::TransformBufferUtilization, + GaugeName::TransformBufferUtilizationLevel, + GaugeName::TransformBufferUtilizationMean, + ), + }; + + let (max_name, legacy_name, max_value) = match (kind, limit) { + (BufferChannelKind::Source, MemoryBufferSize::MaxEvents(n)) => ( + GaugeName::SourceBufferMaxSizeEvents, + GaugeName::SourceBufferMaxEventSize, + n.get() as f64, + ), + (BufferChannelKind::Source, MemoryBufferSize::MaxSize(n)) => ( + GaugeName::SourceBufferMaxSizeBytes, + GaugeName::SourceBufferMaxByteSize, + n.get() as f64, + ), + (BufferChannelKind::Transform, MemoryBufferSize::MaxEvents(n)) => ( + GaugeName::TransformBufferMaxSizeEvents, + GaugeName::TransformBufferMaxEventSize, + n.get() as f64, + ), + (BufferChannelKind::Transform, MemoryBufferSize::MaxSize(n)) => ( + GaugeName::TransformBufferMaxSizeBytes, + GaugeName::TransformBufferMaxByteSize, + n.get() as f64, + ), + }; #[cfg(test)] let recorded_values = Arc::new(Mutex::new(Vec::new())); if let Some(label_value) = output { From 87c287cb84941b9f572408c5d3f520e780225a30 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 18 May 2026 18:04:58 -0400 Subject: [PATCH 43/43] Add transform test --- .../src/topology/channel/limited_queue.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index e2c2eb12fd405..b420396664c20 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -563,6 +563,31 @@ mod tests { assert_eq!(records.last().copied(), Some(0)); } + #[tokio::test] + async fn records_utilization_transform_channel() { + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap()); + let (mut tx, mut rx) = limited( + limit, + Some(ChannelMetricMetadata::new( + BufferChannelKind::Transform, + None, + )), + None, + ); + + let metrics = tx.inner.metrics.as_ref().unwrap().recorded_values.clone(); + + tx.send(Sample::new(1)).await.expect("send should succeed"); + let records = metrics.lock().unwrap().clone(); + assert_eq!(records.len(), 1); + assert_eq!(records.last().copied(), Some(1)); + + assert_eq!(Sample::new(1), rx.next().await.unwrap()); + let records = metrics.lock().unwrap(); + assert_eq!(records.len(), 2); + assert_eq!(records.last().copied(), Some(0)); + } + #[tokio::test] async fn oversized_send_records_true_utilization_via_normal_send_path() { let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap());