chore(metrics): generate internal metric documentation from Rust source

lib/vector-common/src/internal_event/metric_tags.rs

@@ -0,0 +1,230 @@
+use serde_json::{Value, json};
+use std::sync::LazyLock;
+
+/// Clones `base` (a `LazyLock<Value>`) and inserts all fields from `extra`.
+///
+/// Intended for:
+/// - static initializers: `LazyLock::new(|| merge_lazy(&BASE, json!({...})))`
+/// - inline annotations: `merge_lazy(&COMPONENT_TAGS, json!({...}))`
+#[must_use]
+pub fn merge_lazy(base: &LazyLock<Value>, extra: Value) -> Value {
+    let mut result = (**base).clone();
+    if let (Some(obj), Value::Object(extra_obj)) = (result.as_object_mut(), extra) {
+        obj.extend(extra_obj);
+    }
+    result
+}
+
+// ─── Base tag groups ───────────────────────────────────────────────────────────
+
+pub static INTERNAL_METRICS_TAGS: LazyLock<Value> = LazyLock::new(|| {
+    json!({
+        "pid": {"description": "The process ID of the Vector instance.", "required": false, "examples": ["4232"]},
+        "host": {"description": "The hostname of the system Vector is running on.", "required": false, "examples": ["my-host.local"]}
+    })
+});
+
+pub static COMPONENT_TAGS: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &INTERNAL_METRICS_TAGS,
+        json!({
+            "component_kind": {
+                "description": "The Vector component kind.",
+                "required": true,
+                "enum": {
+                    "sink": "Vector sink components",
+                    "source": "Vector source components",
+                    "transform": "Vector transform components"
+                }
+            },
+            "component_id": {"description": "The Vector component ID.", "required": true, "examples": ["my_source", "my_sink"]},
+            "component_type": {"description": "The Vector component type.", "required": true, "examples": ["file", "http", "honeycomb", "splunk_hec"]}
+        }),
+    )
+});
+
+// ─── Extensions of COMPONENT_TAGS ─────────────────────────────────────────────
+
+pub static COMPONENT_TAGS_OUTPUT: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS,
+        json!({
+            "output": {"description": "The specific output of the component.", "required": false}
+        }),
+    )
+});
+
+pub static COMPONENT_TAGS_GRPC_METHOD_SERVICE: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS,
+        json!({
+            "grpc_method": {"description": "The name of the method called on the gRPC service.", "required": true},
+            "grpc_service": {"description": "The gRPC service name.", "required": true}
+        }),
+    )
+});
+
+pub static COMPONENT_TAGS_GRPC_ALL: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS_GRPC_METHOD_SERVICE,
+        json!({
+            "grpc_status": {"description": "The human-readable [gRPC status code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html).", "required": true}
+        }),
+    )
+});
+
+pub static COMPONENT_TAGS_HTTP_METHOD: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS,
+        json!({
+            "method": {"description": "The HTTP method of the request.", "required": false}
+        }),
+    )
+});
+
+pub static COMPONENT_TAGS_HTTP_STATUS: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS,
+        json!({
+            "status": {"description": "The HTTP status code of the request.", "required": false}
+        }),
+    )
+});
+
+pub static COMPONENT_TAGS_HTTP_METHOD_PATH: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS_HTTP_METHOD,
+        json!({
+            "path": {"description": "The path that produced the error.", "required": true}
+        }),
+    )
+});
+
+pub static COMPONENT_TAGS_HTTP_ALL: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS_HTTP_METHOD_PATH,
+        json!({
+            "status": {"description": "The HTTP status code of the request.", "required": false}
+        }),
+    )
+});
+
+pub static COMPONENT_TAGS_ERROR_TYPE_STAGE: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS,
+        json!({
+            "error_type": {
+                "description": "The type of the error",
+                "required": true,
+                "enum": {
+                    "acknowledgements_failed": "The acknowledgement operation failed.",
+                    "delete_failed": "The file deletion failed.",
+                    "encode_failed": "The encode operation failed.",
+                    "field_missing": "The event field was missing.",
+                    "glob_failed": "The glob pattern match operation failed.",
+                    "http_error": "The HTTP request resulted in an error code.",
+                    "invalid_metric": "The metric was invalid.",
+                    "kafka_offset_update": "The consumer offset update failed.",
+                    "kafka_read": "The message from Kafka was invalid.",
+                    "mapping_failed": "The mapping failed.",
+                    "match_failed": "The match operation failed.",
+                    "out_of_order": "The event was out of order.",
+                    "parse_failed": "The parsing operation failed.",
+                    "read_failed": "The file read operation failed.",
+                    "render_error": "The rendering operation failed.",
+                    "stream_closed": "The downstream was closed, forwarding the event(s) failed.",
+                    "type_conversion_failed": "The type conversion operating failed.",
+                    "type_field_does_not_exist": "The type field does not exist.",
+                    "type_ip_address_parse_error": "The IP address did not parse.",
+                    "unlabeled_event": "The event was not labeled.",
+                    "value_invalid": "The value was invalid.",
+                    "watch_failed": "The file watch operation failed.",
+                    "write_failed": "The file write operation failed."
+                }
+            },
+            "stage": {
+                "description": "The stage within the component at which the error occurred.",
+                "required": true,
+                "enum": {
+                    "receiving": "While receiving data.",
+                    "processing": "While processing data within the component.",
+                    "sending": "While sending data."
+                }
+            }
+        }),
+    )
+});
+
+// ─── Extensions of INTERNAL_METRICS_TAGS ──────────────────────────────────────
+
+pub static INTERNAL_METRICS_TAGS_FILE: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &INTERNAL_METRICS_TAGS,
+        json!({
+            "file": {"description": "The file that produced the error.", "required": false}
+        }),
+    )
+});
+
+pub static INTERNAL_METRICS_TAGS_REASON: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &INTERNAL_METRICS_TAGS,
+        json!({
+            "reason": {
+                "description": "The type of the error",
+                "required": true,
+                "enum": {
+                    "out_of_order": "The event was out of order.",
+                    "oversized": "The event was too large."
+                }
+            }
+        }),
+    )
+});
+
+// ─── Metric-specific tag sets ─────────────────────────────────────────────────
+
+pub static COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS,
+        json!({
+            "file": {"description": "The file from which the data originated.", "required": false},
+            "uri": {"description": "The sanitized URI from which the data originated.", "required": false},
+            "container_name": {"description": "The name of the container from which the data originated.", "required": false},
+            "pod_name": {"description": "The name of the pod from which the data originated.", "required": false},
+            "peer_addr": {"description": "The IP from which the data originated.", "required": false},
+            "peer_path": {"description": "The pathname from which the data originated.", "required": false},
+            "mode": {
+                "description": "The connection mode used by the component.",
+                "required": false,
+                "enum": {
+                    "udp": "User Datagram Protocol",
+                    "tcp": "Transmission Control Protocol",
+                    "unix": "Unix domain socket"
+                }
+            }
+        }),
+    )
+});
+
+/// Same tag set as `component_received_events_total` (inherited by byte-count metrics).
+pub static COMPONENT_RECEIVED_EVENTS_TAGS: LazyLock<Value> =
+    LazyLock::new(|| COMPONENT_RECEIVED_EVENTS_TOTAL_TAGS.clone());
+
+pub static COMPONENT_TAGS_HTTP_ERROR_KIND: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS,
+        json!({
+            "error_kind": {"description": "The kind of HTTP error encountered (e.g. connection refused, connection reset).", "required": true}
+        }),
+    )
+});
+
+pub static S3_OBJECT_PROCESSING_TAGS: LazyLock<Value> = LazyLock::new(|| {
+    merge_lazy(
+        &COMPONENT_TAGS,
+        json!({
+            "bucket": {"description": "The name of the S3 bucket.", "required": true}
+        }),
+    )
+});

lib/vector-common/src/internal_event/mod.rs

@@ -6,6 +6,7 @@ pub mod component_events_timed_out;
 mod events_received;
 mod events_sent;
 pub mod metric_name;
+pub mod metric_tags;
 mod optional_tag;
 mod prelude;
 pub mod service;

lib/vector-config-macros/src/ast/mod.rs

@@ -231,6 +231,32 @@ impl FromMeta for Metadata {
                                 }),
                             }
                         }
+                        // Accept path expressions so callers can reference constants:
+                        // `#[configurable(metadata(docs::tags = INTERNAL_METRICS_TAGS))]`
+                        Expr::Path(path) => Some(LazyCustomAttribute::KeyValue {
+                            key: path_to_string(&nv.path),
+                            value: path.to_token_stream(),
+                        }),
+                        // Accept macro invocations such as `serde_json::json!({...})`.
+                        Expr::Macro(mac) => Some(LazyCustomAttribute::KeyValue {
+                            key: path_to_string(&nv.path),
+                            value: mac.to_token_stream(),
+                        }),
+                        // Accept reference/deref expressions such as `&*SOME_LAZY_LOCK`
+                        // so callers can coerce `LazyLock<T>` to `&T: Serialize`.
+                        Expr::Reference(r) => Some(LazyCustomAttribute::KeyValue {
+                            key: path_to_string(&nv.path),
+                            value: r.to_token_stream(),
+                        }),
+                        Expr::Unary(u) => Some(LazyCustomAttribute::KeyValue {
+                            key: path_to_string(&nv.path),
+                            value: u.to_token_stream(),
+                        }),
+                        // Accept function calls such as `merge(&*BASE, json!({...}))`.
+                        Expr::Call(c) => Some(LazyCustomAttribute::KeyValue {
+                            key: path_to_string(&nv.path),
+                            value: c.to_token_stream(),
+                        }),
                         expr => {
                             errors
                                 .push(darling::Error::unexpected_expr_type(expr).with_span(nmeta));

lib/vector-config-macros/src/ast/variant.rs

@@ -1,4 +1,4 @@
-use darling::{FromAttributes, error::Accumulator, util::Flag};
+use darling::{FromAttributes, error::Accumulator, util::Override};
 use proc_macro2::{Ident, TokenStream};
 use quote::ToTokens;
 use serde_derive_internals::ast as serde_ast;
@@ -149,7 +149,20 @@ impl<'a> Variant<'a> {
     /// standard `#[deprecated]` attribute, neither automatically applying it nor deriving the
     /// deprecation status of a variant when it is present.
     pub fn deprecated(&self) -> bool {
-        self.attrs.deprecated.is_present()
+        self.attrs.deprecated.is_some()
+    }
+
+    /// The deprecation message, if one has been set.
+    ///
+    /// Set via `#[configurable(deprecated = "message")]`.
+    pub fn deprecated_message(&self) -> Option<&String> {
+        self.attrs
+            .deprecated
+            .as_ref()
+            .and_then(|message| match message {
+                Override::Inherit => None,
+                Override::Explicit(message) => Some(message),
+            })
     }
 
     /// Whether or not this variant is visible during either serialization or deserialization.
@@ -186,7 +199,7 @@ impl ToTokens for Variant<'_> {
 struct Attributes {
     title: Option<String>,
     description: Option<String>,
-    deprecated: Flag,
+    deprecated: Option<Override<String>>,
     #[darling(skip)]
     visible: bool,
     #[darling(multiple)]

lib/vector-config-macros/src/configurable.rs

@@ -490,6 +490,8 @@ fn generate_variant_metadata(
     let maybe_title = get_metadata_title(meta_ident, variant.title());
     let maybe_description = get_metadata_description(meta_ident, variant.description());
     let maybe_deprecated = get_metadata_deprecated(meta_ident, variant.deprecated());
+    let maybe_deprecated_message =
+        get_metadata_deprecated_message(meta_ident, variant.deprecated_message());
 
     // We have to mark variants as transparent, so that if we're dealing with an untagged enum, we
     // don't panic if their description is intentionally left out.
@@ -520,6 +522,7 @@ fn generate_variant_metadata(
         #maybe_title
         #maybe_description
         #maybe_deprecated
+        #maybe_deprecated_message
         #maybe_transparent
         #maybe_custom_attributes
         #variant_logical_name

src/generate_schema.rs

@@ -3,6 +3,8 @@
 use std::{fs, path::PathBuf};
 
 use clap::Parser;
+use serde_json::{Value, json};
+use vector_common::internal_event::{CounterName, GaugeName, HistogramName};
 use vector_lib::configurable::schema::generate_root_schema;
 
 use crate::config::ConfigBuilder;
@@ -16,11 +18,29 @@ pub struct Opts {
     pub(crate) output_path: Option<PathBuf>,
 }
 
+fn metric_enum_schema<T: vector_lib::configurable::Configurable + 'static>() -> Value {
+    generate_root_schema::<T>()
+        .map(|s| serde_json::to_value(s).unwrap_or(Value::Null))
+        .unwrap_or(Value::Null)
+}
+
 /// Execute the `generate-schema` command.
 #[allow(clippy::print_stdout, clippy::print_stderr)]
 pub fn cmd(opts: &Opts) -> exitcode::ExitCode {
     match generate_root_schema::<ConfigBuilder>() {
-        Ok(schema) => {
+        Ok(config_schema) => {
+            // Convert to Value so we can inject the metric enum schemas.
+            let mut schema = serde_json::to_value(config_schema)
+                .expect("rendering root schema to JSON should not fail");
+
+            // Inject metric name enum schemas so vdev can generate
+            // internal_metrics.cue output descriptions from them.
+            schema["_metric_schemas"] = json!({
+                "counters":   metric_enum_schema::<CounterName>(),
+                "histograms": metric_enum_schema::<HistogramName>(),
+                "gauges":     metric_enum_schema::<GaugeName>(),
+            });
+
             let json = serde_json::to_string_pretty(&schema)
                 .expect("rendering root schema to JSON should not fail");