Merge branch 'arun.pidugu/add-more-fine-grained-controls-tag-cardinality' of github.com:vectordotdev/vector into arun.pidugu/add-more-fine-grained-controls-tag-cardinality

Author: ArunPiduguDD

changelog.d/exclude_cardinality_limit.enhancement.md

@@ -0,0 +1,16 @@
+The `tag_cardinality_limit` transform gained two new configuration capabilities:
+
+- **Per-tag overrides**: each entry in `per_metric_limits` now supports a `per_tag_limits` map
+  whose entries can override `value_limit`, `mode`, and `internal_metrics` for a specific tag
+  key. `limit_exceeded_action` is always inherited from the enclosing per-metric (or global)
+  configuration — it is not per-tag-overridable. Resolution order for the overridable fields is
+  per-tag → per-metric → global.
+- **Exclusion**: `mode: excluded` is now available as a third mode option in `per_metric_limits`
+  and `per_tag_limits` entries (not at the global level). When set, the metric or tag is opted
+  out of cardinality control — all tag values pass through and nothing is tracked. Other
+  tracking fields on the entry (`value_limit`, `internal_metrics`) are ignored when
+  `mode: excluded` is selected. Per-metric exclusion is blanket: when a metric's `mode` is
+  `excluded`, every tag on that metric is excluded and any `per_tag_limits` overrides on it
+  are ignored.
+
+authors: ArunPiduguDD

changelog.d/per_tag_cardinality_limits.enhancement.md

@@ -777,34 +777,4 @@ mod tests {
             }
         }
     }
-
-    fn make_tags(pairs: &[(&str, &str)]) -> MetricTags {
-        pairs
-            .iter()
-            .map(|(k, v)| (k.to_string(), v.to_string()))
-            .collect()
-    }
-
-    #[test]
-    fn rename_with_replacement_basic() {
-        let mut tags = make_tags(&[("old", "v1")]);
-        assert!(tags.rename_with_replacement("old", "new".to_string()));
-        assert_eq!(tags.get("new"), Some("v1"));
-        assert!(!tags.contains_key("old"));
-    }
-
-    #[test]
-    fn rename_with_replacement_missing_old_returns_false() {
-        let mut tags = make_tags(&[("a", "1")]);
-        assert!(!tags.rename_with_replacement("missing", "b".to_string()));
-        assert!(tags.contains_key("a"));
-    }
-
-    #[test]
-    fn rename_with_replacement_overwrites_existing_destination() {
-        let mut tags = make_tags(&[("old", "v1"), ("new", "v2")]);
-        assert!(tags.rename_with_replacement("old", "new".to_string()));
-        assert_eq!(tags.get("new"), Some("v1"));
-        assert!(!tags.contains_key("old"));
-    }
 }

changelog.d/tag_cardinality_limit_per_tag_and_exclude.enhancement.md

@@ -44,60 +44,66 @@ pub struct Config {
     pub per_metric_limits: HashMap<String, PerMetricConfig>,
 }
 
-/// Configuration for the `tag_cardinality_limit` transform for a specific group of metrics.
+/// Configuration block used at the global level.
 #[configurable_component]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub struct Inner {
-    /// How many distinct values to accept for any given key. Ignored when `exclude` is `true`.
+    /// How many distinct values to accept for any given key.
     #[serde(default = "default_value_limit")]
     pub value_limit: usize,
 
     #[configurable(derived)]
     #[serde(default = "default_limit_exceeded_action")]
     pub limit_exceeded_action: LimitExceededAction,
 
-    /// Cardinality tracking mode. Required unless `exclude` is `true`. When both `exclude` and
-    /// `mode` are set, `exclude` takes precedence and `mode` is ignored.
-    #[serde(flatten, default)]
-    pub mode: Option<Mode>,
+    #[serde(flatten)]
+    pub mode: Mode,
 
     #[configurable(derived)]
     #[serde(default)]
     pub internal_metrics: InternalMetricsConfig,
+}
+
+/// Configuration block used at per-metric level. Same shape as the global configuration but
+/// with `OverrideMode`, which adds `excluded` for opting that metric out of cardinality
+/// control entirely.
+#[configurable_component]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct OverrideInner {
+    /// How many distinct values to accept for any given key. Ignored when `mode: excluded`.
+    #[serde(default = "default_value_limit")]
+    pub value_limit: usize,
+
+    #[configurable(derived)]
+    #[serde(default = "default_limit_exceeded_action")]
+    pub limit_exceeded_action: LimitExceededAction,
 
-    /// Exclude this metric or tag from cardinality control entirely. When `true`, all tag values
-    /// pass through and nothing is tracked, regardless of any other fields on this entry.
+    #[serde(flatten)]
+    pub mode: OverrideMode,
+
+    #[configurable(derived)]
     #[serde(default)]
-    pub exclude: bool,
+    pub internal_metrics: InternalMetricsConfig,
 }
 
-impl Inner {
-    /// Validate that at least one of `exclude` or `mode` is set.
-    pub(super) fn validate(&self, location: &str) -> crate::Result<()> {
-        if !self.exclude && self.mode.is_none() {
-            return Err(
-                format!("{location}: `mode` is required unless `exclude: true` is set").into(),
-            );
-        }
-        Ok(())
-    }
-}
+/// Configuration block used at the per-tag level. Same as `OverrideInner` minus
+/// `limit_exceeded_action` — that field is inherited from the enclosing per-metric config
+#[configurable_component]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct PerTagInner {
+    /// How many distinct values to accept for this tag key. Ignored when `mode: excluded`.
+    #[serde(default = "default_value_limit")]
+    pub value_limit: usize,
 
-impl Config {
-    pub(super) fn validate(&self) -> crate::Result<()> {
-        self.global.validate("global")?;
-        for (name, pmc) in &self.per_metric_limits {
-            pmc.config.validate(&format!("per_metric_limits[{name}]"))?;
-            for (tag, ptc) in &pmc.per_tag_limits {
-                ptc.config
-                    .validate(&format!("per_metric_limits[{name}].per_tag_limits[{tag}]"))?;
-            }
-        }
-        Ok(())
-    }
+    #[serde(flatten)]
+    pub mode: OverrideMode,
+
+    #[configurable(derived)]
+    #[serde(default)]
+    pub internal_metrics: InternalMetricsConfig,
 }
 
-/// Controls the approach taken for tracking tag cardinality.
+/// Controls the approach taken for tracking tag cardinality at the global level.
 #[configurable_component]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 #[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)]
@@ -120,6 +126,41 @@ pub enum Mode {
     Probabilistic(BloomFilterConfig),
 }
 
+/// Controls the approach taken for tracking tag cardinality at the per-metric or per-tag level.
+/// Adds `excluded` to the global `Mode` variants.
+#[configurable_component]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)]
+#[configurable(metadata(
+    docs::enum_tag_description = "Controls the approach taken for tracking tag cardinality."
+))]
+pub enum OverrideMode {
+    /// Tracks cardinality exactly. See `Mode::Exact` for details.
+    Exact,
+
+    /// Tracks cardinality probabilistically. See `Mode::Probabilistic` for details.
+    Probabilistic(BloomFilterConfig),
+
+    /// Skip cardinality tracking for this scope. All tag values pass through and nothing is
+    /// recorded. Other tracking fields on the entry (`value_limit`, `limit_exceeded_action`,
+    /// `internal_metrics`) are ignored when this is selected.
+    ///
+    /// Only valid in `per_metric_limits` and `per_tag_limits` entries; using it as the global
+    /// `mode` is a configuration error.
+    Excluded,
+}
+
+impl OverrideMode {
+    /// Returns the equivalent global `Mode` if this scope is tracked, or `None` if excluded.
+    pub const fn as_mode(&self) -> Option<Mode> {
+        match self {
+            OverrideMode::Exact => Some(Mode::Exact),
+            OverrideMode::Probabilistic(b) => Some(Mode::Probabilistic(*b)),
+            OverrideMode::Excluded => None,
+        }
+    }
+}
+
 /// Bloom filter configuration in probabilistic mode.
 #[configurable_component]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -156,9 +197,10 @@ pub struct PerMetricConfig {
 
     /// Per-tag-key overrides scoped to this metric.
     ///
-    /// Each entry has the same fields as a per-metric configuration. When a tag has an entry here,
-    /// that entry replaces the per-metric configuration for that tag. Tags not listed here use this
-    /// per-metric configuration.
+    /// Each entry has the same fields as a per-metric configuration except `limit_exceeded_action`,
+    /// which is always inherited from the enclosing per-metric (or global) configuration. When a
+    /// tag has an entry here, that entry replaces the per-metric `value_limit`, `mode`, and
+    /// `internal_metrics` for that tag. Tags not listed here use the per-metric configuration.
     #[configurable(
         derived,
         metadata(docs::additional_props_description = "An individual tag configuration.")
@@ -167,15 +209,15 @@ pub struct PerMetricConfig {
     pub per_tag_limits: HashMap<String, PerTagConfig>,
 
     #[serde(flatten)]
-    pub config: Inner,
+    pub config: OverrideInner,
 }
 
 /// Tag cardinality limit configuration for a specific tag key, scoped under a per-metric override.
 #[configurable_component]
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub struct PerTagConfig {
     #[serde(flatten)]
-    pub config: Inner,
+    pub config: PerTagInner,
 }
 
 const fn default_limit_exceeded_action() -> LimitExceededAction {
@@ -198,11 +240,10 @@ impl GenerateConfig for Config {
     fn generate_config() -> toml::Value {
         toml::Value::try_from(Self {
             global: Inner {
-                mode: Some(Mode::Exact),
+                mode: Mode::Exact,
                 value_limit: default_value_limit(),
                 limit_exceeded_action: default_limit_exceeded_action(),
                 internal_metrics: InternalMetricsConfig::default(),
-                exclude: false,
             },
             per_metric_limits: HashMap::default(),
         })
@@ -214,7 +255,6 @@ impl GenerateConfig for Config {
 #[typetag::serde(name = "tag_cardinality_limit")]
 impl TransformConfig for Config {
     async fn build(&self, _context: &TransformContext) -> crate::Result<Transform> {
-        self.validate()?;
         Ok(Transform::event_task(TagCardinalityLimit::new(
             self.clone(),
         )))