diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 395dc2b56..5866faead 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4276,6 +4276,16 @@ definitions: description: The condition that the specified config value will be evaluated against anyOf: - "$ref": "#/definitions/ValidateAdheresToSchema" + condition: + title: Condition + description: The condition which will determine if the validation strategy will be applied. + type: string + interpolation_context: + - config + default: "" + examples: + - "{{ config.get('dimensions', False) }}" + - "{{ config.get('custom_reports', [{}])[0].get('dimensions', [])|length > 0 }}" PredicateValidator: title: Predicate Validator description: Validator that applies a validation strategy to a specified value. @@ -4310,6 +4320,16 @@ definitions: description: The validation strategy to apply to the value. anyOf: - "$ref": "#/definitions/ValidateAdheresToSchema" + condition: + title: Condition + description: The condition which will determine if the validation strategy will be applied. + type: string + interpolation_context: + - config + default: "" + examples: + - "{{ config.get('dimensions', False) }}" + - "{{ config.get('custom_reports', [{}])[0].get('dimensions', [])|length > 0 }}" ValidateAdheresToSchema: title: Validate Adheres To Schema description: Validates that a user-provided schema adheres to a specified JSON schema. diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index fad011522..403962d91 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. + # generated by datamodel-codegen: # filename: declarative_component_schema.yaml @@ -2008,6 +2010,15 @@ class DpathValidator(BaseModel): description="The condition that the specified config value will be evaluated against", title="Validation Strategy", ) + condition: Optional[str] = Field( + "", + description="The condition which will determine if the validation strategy will be applied.", + examples=[ + "{{ config.get('dimensions', False) }}", + "{{ config.get('custom_reports', [{}])[0].get('dimensions', [])|length > 0 }}", + ], + title="Condition", + ) class PredicateValidator(BaseModel): @@ -2028,6 +2039,15 @@ class PredicateValidator(BaseModel): description="The validation strategy to apply to the value.", title="Validation Strategy", ) + condition: Optional[str] = Field( + "", + description="The condition which will determine if the validation strategy will be applied.", + examples=[ + "{{ config.get('dimensions', False) }}", + "{{ config.get('custom_reports', [{}])[0].get('dimensions', [])|length > 0 }}", + ], + title="Condition", + ) class ConfigAddFields(BaseModel): diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 081eaca60..977eb1111 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -869,20 +869,26 @@ def create_config_remap_field( def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator: strategy = self._create_component_from_model(model.validation_strategy, config) + condition = model.condition or "" return DpathValidator( field_path=model.field_path, strategy=strategy, + config=config, + condition=condition, ) def create_predicate_validator( self, model: PredicateValidatorModel, config: Config ) -> PredicateValidator: strategy = self._create_component_from_model(model.validation_strategy, config) + condition = model.condition or "" return PredicateValidator( value=model.value, strategy=strategy, + config=config, + condition=condition, ) @staticmethod diff --git a/airbyte_cdk/sources/declarative/validators/dpath_validator.py b/airbyte_cdk/sources/declarative/validators/dpath_validator.py index 05cb12316..24e010c4c 100644 --- a/airbyte_cdk/sources/declarative/validators/dpath_validator.py +++ b/airbyte_cdk/sources/declarative/validators/dpath_validator.py @@ -7,9 +7,11 @@ import dpath.util +from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.validators.validation_strategy import ValidationStrategy from airbyte_cdk.sources.declarative.validators.validator import Validator +from airbyte_cdk.sources.types import Config @dataclass @@ -21,8 +23,12 @@ class DpathValidator(Validator): field_path: List[str] strategy: ValidationStrategy + config: Config + condition: str def __post_init__(self) -> None: + self._interpolated_condition = InterpolatedBoolean(condition=self.condition, parameters={}) + self._field_path = [ InterpolatedString.create(path, parameters={}) for path in self.field_path ] @@ -39,6 +45,9 @@ def validate(self, input_data: dict[str, Any]) -> None: :param input_data: Dictionary containing the data to validate :raises ValueError: If the path doesn't exist or validation fails """ + if self.condition and not self._interpolated_condition.eval(self.config): + return + path = [path.eval({}) for path in self._field_path] if len(path) == 0: diff --git a/airbyte_cdk/sources/declarative/validators/predicate_validator.py b/airbyte_cdk/sources/declarative/validators/predicate_validator.py index 1526777ea..9cd7ed3b9 100644 --- a/airbyte_cdk/sources/declarative/validators/predicate_validator.py +++ b/airbyte_cdk/sources/declarative/validators/predicate_validator.py @@ -5,7 +5,9 @@ from dataclasses import dataclass from typing import Any +from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.validators.validation_strategy import ValidationStrategy +from airbyte_cdk.sources.types import Config @dataclass @@ -16,6 +18,11 @@ class PredicateValidator: value: Any strategy: ValidationStrategy + config: Config + condition: str + + def __post_init__(self) -> None: + self._interpolated_condition = InterpolatedBoolean(condition=self.condition, parameters={}) def validate(self) -> None: """ @@ -23,4 +30,7 @@ def validate(self) -> None: :raises ValueError: If validation fails """ + if self.condition and not self._interpolated_condition.eval(self.config): + return + self.strategy.validate(self.value) diff --git a/unit_tests/sources/declarative/spec/test_spec.py b/unit_tests/sources/declarative/spec/test_spec.py index 75287082c..1461583fb 100644 --- a/unit_tests/sources/declarative/spec/test_spec.py +++ b/unit_tests/sources/declarative/spec/test_spec.py @@ -210,6 +210,7 @@ def test_given_list_of_transformations_when_transform_config_then_config_is_tran def test_given_valid_config_value_when_validating_then_no_exception_is_raised() -> None: + input_config = {"test_field": {"field_to_validate": "test"}} spec = component_spec( connection_specification={}, parameters={}, @@ -233,14 +234,16 @@ def test_given_valid_config_value_when_validating_then_no_exception_is_raised() }, } ), + config=input_config, + condition="", ) ], ) - input_config = {"test_field": {"field_to_validate": "test"}} spec.validate_config(input_config) def test_given_invalid_config_value_when_validating_then_exception_is_raised() -> None: + input_config = {"test_field": {"field_to_validate": 123}} spec = component_spec( connection_specification={}, parameters={}, @@ -263,10 +266,11 @@ def test_given_invalid_config_value_when_validating_then_exception_is_raised() - }, } ), + config=input_config, + condition="", ) ], ) - input_config = {"test_field": {"field_to_validate": 123}} with pytest.raises(Exception): spec.validate_config(input_config) diff --git a/unit_tests/sources/declarative/validators/test_dpath_validator.py b/unit_tests/sources/declarative/validators/test_dpath_validator.py index c5426e351..950921028 100644 --- a/unit_tests/sources/declarative/validators/test_dpath_validator.py +++ b/unit_tests/sources/declarative/validators/test_dpath_validator.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. + from unittest import TestCase import pytest @@ -23,7 +25,12 @@ def validate(self, value): class TestDpathValidator(TestCase): def test_given_valid_path_and_input_validate_is_successful(self): strategy = MockValidationStrategy() - validator = DpathValidator(field_path=["user", "profile", "email"], strategy=strategy) + validator = DpathValidator( + field_path=["user", "profile", "email"], + strategy=strategy, + config={}, + condition="", + ) test_data = {"user": {"profile": {"email": "test@example.com", "name": "Test User"}}} @@ -34,7 +41,12 @@ def test_given_valid_path_and_input_validate_is_successful(self): def test_given_invalid_path_when_validate_then_raise_key_error(self): strategy = MockValidationStrategy() - validator = DpathValidator(field_path=["user", "profile", "phone"], strategy=strategy) + validator = DpathValidator( + field_path=["user", "profile", "phone"], + strategy=strategy, + config={}, + condition="", + ) test_data = {"user": {"profile": {"email": "test@example.com"}}} @@ -47,7 +59,12 @@ def test_given_invalid_path_when_validate_then_raise_key_error(self): def test_given_strategy_fails_when_validate_then_raise_value_error(self): error_message = "Invalid email format" strategy = MockValidationStrategy(should_fail=True, error_message=error_message) - validator = DpathValidator(field_path=["user", "email"], strategy=strategy) + validator = DpathValidator( + field_path=["user", "email"], + strategy=strategy, + config={}, + condition="", + ) test_data = {"user": {"email": "invalid-email"}} @@ -59,7 +76,12 @@ def test_given_strategy_fails_when_validate_then_raise_value_error(self): def test_given_empty_path_list_when_validate_then_validate_raises_exception(self): strategy = MockValidationStrategy() - validator = DpathValidator(field_path=[], strategy=strategy) + validator = DpathValidator( + field_path=[], + strategy=strategy, + config={}, + condition="", + ) test_data = {"key": "value"} with pytest.raises(ValueError): @@ -67,7 +89,12 @@ def test_given_empty_path_list_when_validate_then_validate_raises_exception(self def test_given_empty_input_data_when_validate_then_validate_raises_exception(self): strategy = MockValidationStrategy() - validator = DpathValidator(field_path=["data", "field"], strategy=strategy) + validator = DpathValidator( + field_path=["data", "field"], + strategy=strategy, + config={}, + condition="", + ) test_data = {} @@ -76,7 +103,12 @@ def test_given_empty_input_data_when_validate_then_validate_raises_exception(sel def test_path_with_wildcard_when_validate_then_validate_is_successful(self): strategy = MockValidationStrategy() - validator = DpathValidator(field_path=["users", "*", "email"], strategy=strategy) + validator = DpathValidator( + field_path=["users", "*", "email"], + strategy=strategy, + config={}, + condition="", + ) test_data = { "users": { @@ -90,3 +122,16 @@ def test_path_with_wildcard_when_validate_then_validate_is_successful(self): assert strategy.validate_called assert strategy.validated_value in ["user1@example.com", "user2@example.com"] self.assertIn(strategy.validated_value, ["user1@example.com", "user2@example.com"]) + + def test_given_condition_is_false_when_validate_then_validate_is_not_called(self): + strategy = MockValidationStrategy() + validator = DpathValidator( + field_path=["user", "profile", "email"], + strategy=strategy, + config={"test": "test"}, + condition="{{ not config.get('test') }}", + ) + + validator.validate({"user": {"profile": {"email": "test@example.com"}}}) + + assert not strategy.validate_called diff --git a/unit_tests/sources/declarative/validators/test_predicate_validator.py b/unit_tests/sources/declarative/validators/test_predicate_validator.py index 1f0b3b7c0..f7a39ff35 100644 --- a/unit_tests/sources/declarative/validators/test_predicate_validator.py +++ b/unit_tests/sources/declarative/validators/test_predicate_validator.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. + from unittest import TestCase import pytest @@ -24,7 +26,7 @@ class TestPredicateValidator(TestCase): def test_given_valid_input_validate_is_successful(self): strategy = MockValidationStrategy() test_value = "test@example.com" - validator = PredicateValidator(value=test_value, strategy=strategy) + validator = PredicateValidator(value=test_value, strategy=strategy, config={}, condition="") validator.validate() @@ -35,7 +37,7 @@ def test_given_invalid_input_when_validate_then_raise_value_error(self): error_message = "Invalid email format" strategy = MockValidationStrategy(should_fail=True, error_message=error_message) test_value = "invalid-email" - validator = PredicateValidator(value=test_value, strategy=strategy) + validator = PredicateValidator(value=test_value, strategy=strategy, config={}, condition="") with pytest.raises(ValueError) as context: validator.validate() @@ -47,9 +49,22 @@ def test_given_invalid_input_when_validate_then_raise_value_error(self): def test_given_complex_object_when_validate_then_successful(self): strategy = MockValidationStrategy() test_value = {"user": {"email": "test@example.com", "name": "Test User"}} - validator = PredicateValidator(value=test_value, strategy=strategy) + validator = PredicateValidator(value=test_value, strategy=strategy, config={}, condition="") validator.validate() assert strategy.validate_called assert strategy.validated_value == test_value + + def test_given_condition_is_false_when_validate_then_validate_is_not_called(self): + strategy = MockValidationStrategy() + validator = PredicateValidator( + value="test", + strategy=strategy, + config={"test": "test"}, + condition="{{ not config.get('test') }}", + ) + + validator.validate() + + assert not strategy.validate_called