Skip to content

Commit 6e1c15c

Browse files
committed
fix(cli): validate transform VRL and conditions with --no-environment
vector validate --no-environment now compiles VRL programs (remap) and conditions (filter) to catch expression errors without requiring environment access. Previously these errors were only caught when running without the flag. TransformConfig::validate now accepts a TransformContext instead of a bare schema::Definition, so transforms can access enrichment_tables, metrics_storage, and the merged schema definition for type-aware VRL compilation. Environment-dependent transforms (aws_ec2_metadata) use the default no-op validate impl and are unaffected. Closes: #15037
1 parent 2d0d0fb commit 6e1c15c

11 files changed

Lines changed: 193 additions & 22 deletions

File tree

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fixed `vector validate --no-environment` so it reports VRL and condition compilation errors for transforms without requiring full environment-dependent component initialization.
2+
3+
authors: pront

src/conditions/mod.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,14 @@ impl AnyCondition {
210210
AnyCondition::Map(m) => m.build(enrichment_tables, metrics_storage),
211211
}
212212
}
213+
214+
pub fn validate(
215+
&self,
216+
enrichment_tables: &vector_lib::enrichment::TableRegistry,
217+
metrics_storage: &MetricsStorage,
218+
) -> crate::Result<()> {
219+
self.build(enrichment_tables, metrics_storage).map(|_| ())
220+
}
213221
}
214222

215223
impl From<ConditionConfig> for AnyCondition {

src/config/transform.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ pub struct TransformContext {
162162
/// (e.g. `aws_ec2_metadata`, `throttle`) clone this and pass it to [`crate::cpu_time::spawn_timed`] so
163163
/// their CPU is attributed to the component alongside the main transform task.
164164
pub cpu_ns: Option<Counter>,
165+
165166
}
166167

167168
impl Default for TransformContext {
@@ -250,7 +251,7 @@ pub trait TransformConfig: DynClone + NamedComponent + core::fmt::Debug + Send +
250251
///
251252
/// If validation does not succeed, an error variant containing a list of all validation errors
252253
/// is returned.
253-
fn validate(&self, _merged_definition: &schema::Definition) -> Result<(), Vec<String>> {
254+
fn validate(&self, _context: &TransformContext) -> Result<(), Vec<String>> {
254255
Ok(())
255256
}
256257

src/config/validation.rs

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use super::{
99
ComponentKey, Config, OutputId, Resource, builder::ConfigBuilder,
1010
transform::get_transform_output_ids,
1111
};
12-
use crate::config::schema;
12+
use crate::config::TransformContext;
1313

1414
/// Minimum value (exclusive) for EWMA alpha options.
1515
/// The alpha value must be strictly greater than this value.
@@ -234,10 +234,7 @@ pub fn check_outputs(config: &ConfigBuilder) -> Result<(), Vec<String>> {
234234
}
235235

236236
for (key, transform) in config.transforms.iter() {
237-
// use the most general definition possible, since the real value isn't known yet.
238-
let definition = schema::Definition::any();
239-
240-
if let Err(errs) = transform.inner.validate(&definition) {
237+
if let Err(errs) = transform.inner.validate(&TransformContext::default()) {
241238
errors.extend(errs.into_iter().map(|msg| format!("Transform {key} {msg}")));
242239
}
243240

src/transforms/exclusive_route/config.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ impl TransformConfig for ExclusiveRouteConfig {
101101
Input::all()
102102
}
103103

104-
fn validate(&self, _: &schema::Definition) -> Result<(), Vec<String>> {
104+
fn validate(&self, _: &TransformContext) -> Result<(), Vec<String>> {
105105
let mut errors = Vec::new();
106106

107107
let mut counts = std::collections::HashMap::new();

src/transforms/filter.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,12 @@ impl TransformConfig for FilterConfig {
5050
)?)))
5151
}
5252

53+
fn validate(&self, context: &TransformContext) -> Result<(), Vec<String>> {
54+
self.condition
55+
.validate(&context.enrichment_tables, &context.metrics_storage)
56+
.map_err(|e| vec![e.to_string()])
57+
}
58+
5359
fn input(&self) -> Input {
5460
Input::all()
5561
}

src/transforms/remap.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,16 @@ impl TransformConfig for RemapConfig {
277277
Ok(transform)
278278
}
279279

280+
fn validate(&self, context: &TransformContext) -> std::result::Result<(), Vec<String>> {
281+
self.compile_vrl_program(
282+
context.enrichment_tables.clone(),
283+
context.metrics_storage.clone(),
284+
context.merged_schema_definition.clone(),
285+
)
286+
.map(|_| ())
287+
.map_err(|e| vec![e.to_string()])
288+
}
289+
280290
fn input(&self) -> Input {
281291
Input::all()
282292
}

src/transforms/route.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ impl TransformConfig for RouteConfig {
131131
Input::all()
132132
}
133133

134-
fn validate(&self, _: &schema::Definition) -> Result<(), Vec<String>> {
134+
fn validate(&self, _: &TransformContext) -> Result<(), Vec<String>> {
135135
if self.route.contains_key(UNMATCHED_ROUTE) {
136136
Err(vec![format!(
137137
"cannot have a named output with reserved name: `{UNMATCHED_ROUTE}`"

src/transforms/sample/config.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ impl TransformConfig for SampleConfig {
222222
Input::new(DataType::Log | DataType::Trace)
223223
}
224224

225-
fn validate(&self, _: &schema::Definition) -> Result<(), Vec<String>> {
225+
fn validate(&self, _: &TransformContext) -> Result<(), Vec<String>> {
226226
self.sample_rate()
227227
.map(|_| ())
228228
.map_err(|e| vec![e.to_string()])
@@ -317,7 +317,7 @@ mod tests {
317317
exclude: None,
318318
};
319319

320-
assert!(config.validate(&crate::schema::Definition::any()).is_ok());
320+
assert!(config.validate(&crate::config::TransformContext::default()).is_ok());
321321
}
322322

323323
#[test]

src/validate.rs

Lines changed: 59 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
#![allow(missing_docs)]
22

3-
use std::{fmt, fs::remove_dir_all, path::PathBuf};
3+
use std::{collections::HashMap, fmt, fs::remove_dir_all, path::PathBuf};
44

55
use clap::Parser;
66
use colored::*;
77
use exitcode::ExitCode;
8+
use vector_vrl_metrics::MetricsStorage;
89

910
use crate::{
10-
config::{self, Config, ConfigDiff, loading::ConfigBuilderLoader},
11+
config::{self, Config, ConfigDiff, TransformContext, loading::ConfigBuilderLoader},
12+
schema::Definition,
1113
topology::{
1214
self,
1315
builder::{TopologyPieces, TopologyPiecesBuilder},
@@ -117,13 +119,13 @@ pub async fn validate(opts: &Opts, color: bool) -> ExitCode {
117119
None => return exitcode::CONFIG,
118120
};
119121

120-
if !opts.no_environment {
121-
if let Some(tmp_directory) = create_tmp_directory(&mut config, &mut fmt) {
122-
validated &= validate_environment(opts, &config, &mut fmt).await;
123-
remove_tmp_directory(tmp_directory);
124-
} else {
125-
validated = false;
126-
}
122+
if opts.no_environment {
123+
validated &= validate_transforms_no_environment(&config, &mut fmt).await;
124+
} else if let Some(tmp_directory) = create_tmp_directory(&mut config, &mut fmt) {
125+
validated &= validate_environment(opts, &config, &mut fmt).await;
126+
remove_tmp_directory(tmp_directory);
127+
} else {
128+
validated = false;
127129
}
128130

129131
if validated {
@@ -180,6 +182,54 @@ pub fn validate_config(opts: &Opts, fmt: &mut Formatter) -> Option<Config> {
180182
Some(config)
181183
}
182184

185+
async fn validate_transforms_no_environment(config: &Config, fmt: &mut Formatter) -> bool {
186+
let enrichment_tables = vector_lib::enrichment::TableRegistry::default();
187+
let metrics_storage = MetricsStorage::default();
188+
let mut definition_cache = HashMap::new();
189+
let mut errors = Vec::new();
190+
191+
for (key, transform) in config.transforms() {
192+
let input_definitions = topology::schema::input_definitions(
193+
&transform.inputs,
194+
config,
195+
enrichment_tables.clone(),
196+
&mut definition_cache,
197+
)
198+
.unwrap_or_default();
199+
200+
let merged_schema_definition = input_definitions
201+
.iter()
202+
.map(|(_, definition)| definition.clone())
203+
.reduce(Definition::merge)
204+
.unwrap_or_else(Definition::any);
205+
206+
let context = TransformContext {
207+
key: Some(key.clone()),
208+
globals: config.global.clone(),
209+
enrichment_tables: enrichment_tables.clone(),
210+
metrics_storage: metrics_storage.clone(),
211+
merged_schema_definition,
212+
schema: config.schema,
213+
..Default::default()
214+
};
215+
216+
if let Err(errs) = transform.inner.validate(&context) {
217+
for err in errs {
218+
errors.push(format!("Transform \"{key}\": {err}"));
219+
}
220+
}
221+
}
222+
223+
if errors.is_empty() {
224+
fmt.success("Transform configuration");
225+
true
226+
} else {
227+
fmt.title("Transform errors");
228+
fmt.sub_error(errors);
229+
false
230+
}
231+
}
232+
183233
async fn validate_environment(opts: &Opts, config: &Config, fmt: &mut Formatter) -> bool {
184234
let diff = ConfigDiff::initial(config);
185235

0 commit comments

Comments
 (0)