Skip to content

Commit 5465781

Browse files
committed
fix(cli): validate transform VRL and conditions with --no-environment
vector validate --no-environment now catches VRL compilation errors and invalid condition expressions in transforms before any environment resources are available. Design: - Add validate(&TransformContext) to TransformConfig; remap and filter implement it to compile VRL programs and conditions respectively - Introduce validate_transforms(), called in both --no-environment and normal validate paths, which builds stub enrichment tables from the config so VRL can resolve table name references without loading data - Remove the validate() call from check_outputs: that function is for graph/wiring checks only; transform compilation belongs in validate_transforms() - Convert all CLI integration test configs to YAML; add create_yaml_file helper so test files carry the .yaml extension Vector uses for format detection Fixes #15037
1 parent 6289586 commit 5465781

36 files changed

Lines changed: 348 additions & 84 deletions
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fixed `vector validate --no-environment` so it reports VRL and condition compilation errors for transforms without requiring full environment-dependent component initialization.
2+
3+
authors: pront

lib/codecs/src/decoding/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,7 @@ pub enum DeserializerConfig {
311311
/// [influxdb]: https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol
312312
Influxdb(InfluxdbDeserializerConfig),
313313

314-
/// Decodes the raw bytes as an [Apache Avro][apache_avro] message.
314+
/// Decodes the raw bytes as as an [Apache Avro][apache_avro] message.
315315
///
316316
/// [apache_avro]: https://avro.apache.org/
317317
Avro {

src/conditions/mod.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,14 @@ impl AnyCondition {
210210
AnyCondition::Map(m) => m.build(enrichment_tables, metrics_storage),
211211
}
212212
}
213+
214+
pub fn validate(
215+
&self,
216+
enrichment_tables: &vector_lib::enrichment::TableRegistry,
217+
metrics_storage: &MetricsStorage,
218+
) -> crate::Result<()> {
219+
self.build(enrichment_tables, metrics_storage).map(|_| ())
220+
}
213221
}
214222

215223
impl From<ConditionConfig> for AnyCondition {

src/config/transform.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ pub trait TransformConfig: DynClone + NamedComponent + core::fmt::Debug + Send +
250250
///
251251
/// If validation does not succeed, an error variant containing a list of all validation errors
252252
/// is returned.
253-
fn validate(&self, _merged_definition: &schema::Definition) -> Result<(), Vec<String>> {
253+
fn validate(&self, _context: &TransformContext) -> Result<(), Vec<String>> {
254254
Ok(())
255255
}
256256

src/config/validation.rs

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ use super::{
99
ComponentKey, Config, OutputId, Resource, builder::ConfigBuilder,
1010
transform::get_transform_output_ids,
1111
};
12-
use crate::config::schema;
1312

1413
/// Minimum value (exclusive) for EWMA alpha options.
1514
/// The alpha value must be strictly greater than this value.
@@ -234,13 +233,6 @@ pub fn check_outputs(config: &ConfigBuilder) -> Result<(), Vec<String>> {
234233
}
235234

236235
for (key, transform) in config.transforms.iter() {
237-
// use the most general definition possible, since the real value isn't known yet.
238-
let definition = schema::Definition::any();
239-
240-
if let Err(errs) = transform.inner.validate(&definition) {
241-
errors.extend(errs.into_iter().map(|msg| format!("Transform {key} {msg}")));
242-
}
243-
244236
if get_transform_output_ids(
245237
transform.inner.as_ref(),
246238
key.clone(),

src/transforms/exclusive_route/config.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ impl TransformConfig for ExclusiveRouteConfig {
101101
Input::all()
102102
}
103103

104-
fn validate(&self, _: &schema::Definition) -> Result<(), Vec<String>> {
104+
fn validate(&self, _: &TransformContext) -> Result<(), Vec<String>> {
105105
let mut errors = Vec::new();
106106

107107
let mut counts = std::collections::HashMap::new();

src/transforms/filter.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,12 @@ impl TransformConfig for FilterConfig {
5050
)?)))
5151
}
5252

53+
fn validate(&self, context: &TransformContext) -> Result<(), Vec<String>> {
54+
self.condition
55+
.validate(&context.enrichment_tables, &context.metrics_storage)
56+
.map_err(|e| vec![e.to_string()])
57+
}
58+
5359
fn input(&self) -> Input {
5460
Input::all()
5561
}

src/transforms/remap.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,16 @@ impl TransformConfig for RemapConfig {
277277
Ok(transform)
278278
}
279279

280+
fn validate(&self, context: &TransformContext) -> std::result::Result<(), Vec<String>> {
281+
self.compile_vrl_program(
282+
context.enrichment_tables.clone(),
283+
context.metrics_storage.clone(),
284+
context.merged_schema_definition.clone(),
285+
)
286+
.map(|_| ())
287+
.map_err(|e| vec![e.to_string()])
288+
}
289+
280290
fn input(&self) -> Input {
281291
Input::all()
282292
}

src/transforms/route.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ impl TransformConfig for RouteConfig {
131131
Input::all()
132132
}
133133

134-
fn validate(&self, _: &schema::Definition) -> Result<(), Vec<String>> {
134+
fn validate(&self, _: &TransformContext) -> Result<(), Vec<String>> {
135135
if self.route.contains_key(UNMATCHED_ROUTE) {
136136
Err(vec![format!(
137137
"cannot have a named output with reserved name: `{UNMATCHED_ROUTE}`"

src/transforms/sample/config.rs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ impl TransformConfig for SampleConfig {
222222
Input::new(DataType::Log | DataType::Trace)
223223
}
224224

225-
fn validate(&self, _: &schema::Definition) -> Result<(), Vec<String>> {
225+
fn validate(&self, _: &TransformContext) -> Result<(), Vec<String>> {
226226
self.sample_rate()
227227
.map(|_| ())
228228
.map_err(|e| vec![e.to_string()])
@@ -317,7 +317,11 @@ mod tests {
317317
exclude: None,
318318
};
319319

320-
assert!(config.validate(&crate::schema::Definition::any()).is_ok());
320+
assert!(
321+
config
322+
.validate(&crate::config::TransformContext::default())
323+
.is_ok()
324+
);
321325
}
322326

323327
#[test]

0 commit comments

Comments
 (0)