From 78dbcc2906b00614b645c7ad9dd7b77e96ea7379 Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 30 Jun 2025 14:41:51 -0700 Subject: [PATCH 1/7] switch query field to string --- .../declarative_component_schema.yaml | 4 +- .../models/declarative_component_schema.py | 120 +++++++++++------- 2 files changed, 73 insertions(+), 51 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 305b351bc..ac00187cc 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4326,10 +4326,8 @@ definitions: - query properties: query: - type: object - additionalProperties: true + type: string description: The GraphQL query to be executed - default: {} additionalProperties: true DpathValidator: title: Dpath Validator diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index d7328d0ce..519e82f2d 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -618,7 +618,9 @@ class OAuthAuthenticator(BaseModel): scopes: Optional[List[str]] = Field( None, description="List of scopes that should be granted to the access token.", - examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], + examples=[ + ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] + ], title="Scopes", ) token_expiry_date: Optional[str] = Field( @@ -1124,24 +1126,28 @@ class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], - } - }, - ], - title="OAuth user input", + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( + Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", + ) ) - oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( - None, - description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', - title="DeclarativeOAuth Connector Specification", + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( + Field( + None, + description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', + title="DeclarativeOAuth Connector Specification", + ) ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, @@ -1159,7 +1165,9 @@ class Config: complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], + examples=[ + {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} + ], title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( @@ -1554,7 +1562,7 @@ class RequestBodyGraphQlQuery(BaseModel): class Config: extra = Extra.allow - query: Dict[str, Any] = Field(..., description="The GraphQL query to be executed") + query: str = Field(..., description="The GraphQL query to be executed") class ValidateAdheresToSchema(BaseModel): @@ -2005,7 +2013,9 @@ class RecordSelector(BaseModel): description="Responsible for filtering records to be emitted by the Source.", title="Record Filter", ) - schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( + schema_normalization: Optional[ + Union[SchemaNormalization, CustomSchemaNormalization] + ] = Field( None, description="Responsible for normalization according to the schema.", title="Schema Normalization", @@ -2041,10 +2051,12 @@ class DpathValidator(BaseModel): ], title="Field Path", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The condition that the specified config value will be evaluated against", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The condition that the specified config value will be evaluated against", + title="Validation Strategy", + ) ) @@ -2061,10 +2073,12 @@ class PredicateValidator(BaseModel): ], title="Value", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The validation strategy to apply to the value.", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The validation strategy to apply to the value.", + title="Validation Strategy", + ) ) @@ -2147,7 +2161,9 @@ class ConfigMigration(BaseModel): description: Optional[str] = Field( None, description="The description/purpose of the config migration." ) - transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field( + transformations: List[ + Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields] + ] = Field( ..., description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", title="Transformations", @@ -2239,9 +2255,9 @@ class Config: type: Literal["DeclarativeSource"] check: Union[CheckStream, CheckDynamicStream] - streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( - None - ) + streams: Optional[ + List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] + ] = None dynamic_streams: List[DynamicDeclarativeStream] version: str = Field( ..., @@ -2366,7 +2382,9 @@ class Config: extra = Extra.allow type: Literal["DeclarativeStream"] - name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + name: Optional[str] = Field( + "", description="The stream name.", example=["Users"], title="Name" + ) retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages.", @@ -2544,18 +2562,20 @@ class HttpRequester(BaseModelWithDeprecations): description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", title="Fetch Properties from Endpoint", ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( + Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", + ) ) request_headers: Optional[Union[Dict[str, str], str]] = Field( None, @@ -2750,7 +2770,9 @@ class QueryProperties(BaseModel): class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] - name: str = Field(..., description="The stream name.", example=["Users"], title="Name") + name: str = Field( + ..., description="The stream name.", example=["Users"], title="Name" + ) full_refresh_stream: DeclarativeStream = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", @@ -2839,7 +2861,9 @@ class AsyncRetriever(BaseModel): ) download_extractor: Optional[ Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] - ] = Field(None, description="Responsible for fetching the records from provided urls.") + ] = Field( + None, description="Responsible for fetching the records from provided urls." + ) creation_requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", From 623f0e77866a4be71526874b9afa23a989002ce2 Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 30 Jun 2025 15:06:18 -0700 Subject: [PATCH 2/7] fix graphql request body interpolation and conversion --- .../interpolated_request_options_provider.py | 2 +- unit_tests/manifest_migrations/conftest.py | 13 +++---------- .../test_interpolated_request_options_provider.py | 4 ++-- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py b/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py index 97ef78d48..ade3b99ac 100644 --- a/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +++ b/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py @@ -100,7 +100,7 @@ def _resolve_request_body(self) -> None: if self.request_body.type == "RequestBodyUrlEncodedForm": self.request_body_data = self.request_body.value elif self.request_body.type == "RequestBodyGraphQL": - self.request_body_json = {"query": self.request_body.value.query} + self.request_body_json = self.request_body.value.dict() elif self.request_body.type in ("RequestBodyJsonObject", "RequestBodyPlainText"): self.request_body_json = self.request_body.value else: diff --git a/unit_tests/manifest_migrations/conftest.py b/unit_tests/manifest_migrations/conftest.py index 14caf8865..afb93b8be 100644 --- a/unit_tests/manifest_migrations/conftest.py +++ b/unit_tests/manifest_migrations/conftest.py @@ -741,8 +741,7 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic # this example holds the GraphQL query object. "request_body_json": { "query": { - "field": "{{ config['query_field'] }}", - "value": "{{ config['query_value'] }}", + "query { {{ config['query_field'] }} { {{ config['query_value'] }} }}", } }, }, @@ -975,10 +974,7 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]: "request_body": { "type": "RequestBodyGraphQL", "value": { - "query": { - "field": "{{ config['query_field'] }}", - "value": "{{ config['query_value'] }}", - } + "query": "query { {{ config['query_field'] }} { {{ config['query_value'] }} }}", }, }, }, @@ -1138,10 +1134,7 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]: "request_body": { "type": "RequestBodyGraphQL", "value": { - "query": { - "field": "{{ config['query_field'] }}", - "value": "{{ config['query_value'] }}", - } + "query": "query { {{ config['query_field'] }} { {{ config['query_value'] }} }}", }, }, }, diff --git a/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py b/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py index b112791e7..075ee7150 100644 --- a/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +++ b/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py @@ -239,10 +239,10 @@ def test_interpolated_request_json(test_name, input_request_json, expected_reque RequestBodyGraphQL( type="RequestBodyGraphQL", value=RequestBodyGraphQlQuery( - query={"query_key": "{{ config['option'] }}", "query_key_2": "value"} + query="query { {{ config['option'] }} }" ), ), - {"query": {"query_key": "OPTION", "query_key_2": "value"}}, + {"query": "query { OPTION }"}, ), ], ) From 27f20576f839185481ba4e0e740a6654ece67a8b Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 30 Jun 2025 15:15:25 -0700 Subject: [PATCH 3/7] fix migration and unit tests, and add variables to unit tests --- ...ttp_requester_request_body_json_data_to_request_body.py | 2 +- unit_tests/manifest_migrations/conftest.py | 7 ++++--- .../test_interpolated_request_options_provider.py | 5 +++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py b/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py index c63066953..9ff2e9018 100644 --- a/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py +++ b/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py @@ -54,7 +54,7 @@ def _migrate_body_json(self, manifest: ManifestType, key: str) -> None: if isinstance(manifest[key], str): self._migrate_value(manifest, key, text_type) elif isinstance(manifest[key], dict): - if manifest[key].get(query_key) is not None: + if isinstance(manifest[key].get(query_key), str): self._migrate_value(manifest, key, graph_ql_type) else: self._migrate_value(manifest, key, json_object_type) diff --git a/unit_tests/manifest_migrations/conftest.py b/unit_tests/manifest_migrations/conftest.py index afb93b8be..33bc5317f 100644 --- a/unit_tests/manifest_migrations/conftest.py +++ b/unit_tests/manifest_migrations/conftest.py @@ -740,9 +740,8 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic # the `request_body_json` is expected to be migrated to the `request_body` key, # this example holds the GraphQL query object. "request_body_json": { - "query": { - "query { {{ config['query_field'] }} { {{ config['query_value'] }} }}", - } + "query": "query { {{ config['query_field'] }} { {{ config['query_value'] }} }}", + "variables": {"arg1": "test"} }, }, "record_selector": { @@ -975,6 +974,7 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]: "type": "RequestBodyGraphQL", "value": { "query": "query { {{ config['query_field'] }} { {{ config['query_value'] }} }}", + "variables": {"arg1": "test"}, }, }, }, @@ -1135,6 +1135,7 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]: "type": "RequestBodyGraphQL", "value": { "query": "query { {{ config['query_field'] }} { {{ config['query_value'] }} }}", + "variables": {"arg1": "test"}, }, }, }, diff --git a/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py b/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py index 075ee7150..786807aa6 100644 --- a/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +++ b/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py @@ -239,10 +239,11 @@ def test_interpolated_request_json(test_name, input_request_json, expected_reque RequestBodyGraphQL( type="RequestBodyGraphQL", value=RequestBodyGraphQlQuery( - query="query { {{ config['option'] }} }" + query="query { {{ config['option'] }} }", + variables={"startDate": "{{ stream_interval['start_date'] }}"}, ), ), - {"query": "query { OPTION }"}, + {"query": "query { OPTION }", "variables": {"startDate": "2020-01-01"}}, ), ], ) From 7b5aadbeff662b9515b5e537343cefc7e84b2e45 Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 30 Jun 2025 15:22:38 -0700 Subject: [PATCH 4/7] ruff format --- .../models/declarative_component_schema.py | 118 +++++++----------- unit_tests/manifest_migrations/conftest.py | 2 +- 2 files changed, 48 insertions(+), 72 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 519e82f2d..71a517774 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -618,9 +618,7 @@ class OAuthAuthenticator(BaseModel): scopes: Optional[List[str]] = Field( None, description="List of scopes that should be granted to the access token.", - examples=[ - ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] - ], + examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], title="Scopes", ) token_expiry_date: Optional[str] = Field( @@ -1126,28 +1124,24 @@ class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( - Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], - } - }, - ], - title="OAuth user input", - ) + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", ) - oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( - Field( - None, - description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', - title="DeclarativeOAuth Connector Specification", - ) + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( + None, + description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', + title="DeclarativeOAuth Connector Specification", ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, @@ -1165,9 +1159,7 @@ class Config: complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[ - {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} - ], + examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( @@ -2013,9 +2005,7 @@ class RecordSelector(BaseModel): description="Responsible for filtering records to be emitted by the Source.", title="Record Filter", ) - schema_normalization: Optional[ - Union[SchemaNormalization, CustomSchemaNormalization] - ] = Field( + schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( None, description="Responsible for normalization according to the schema.", title="Schema Normalization", @@ -2051,12 +2041,10 @@ class DpathValidator(BaseModel): ], title="Field Path", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( - Field( - ..., - description="The condition that the specified config value will be evaluated against", - title="Validation Strategy", - ) + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( + ..., + description="The condition that the specified config value will be evaluated against", + title="Validation Strategy", ) @@ -2073,12 +2061,10 @@ class PredicateValidator(BaseModel): ], title="Value", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( - Field( - ..., - description="The validation strategy to apply to the value.", - title="Validation Strategy", - ) + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( + ..., + description="The validation strategy to apply to the value.", + title="Validation Strategy", ) @@ -2161,9 +2147,7 @@ class ConfigMigration(BaseModel): description: Optional[str] = Field( None, description="The description/purpose of the config migration." ) - transformations: List[ - Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields] - ] = Field( + transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field( ..., description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.", title="Transformations", @@ -2255,9 +2239,9 @@ class Config: type: Literal["DeclarativeSource"] check: Union[CheckStream, CheckDynamicStream] - streams: Optional[ - List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] - ] = None + streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( + None + ) dynamic_streams: List[DynamicDeclarativeStream] version: str = Field( ..., @@ -2382,9 +2366,7 @@ class Config: extra = Extra.allow type: Literal["DeclarativeStream"] - name: Optional[str] = Field( - "", description="The stream name.", example=["Users"], title="Name" - ) + name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages.", @@ -2562,20 +2544,18 @@ class HttpRequester(BaseModelWithDeprecations): description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", title="Fetch Properties from Endpoint", ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( - Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", - ) + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", ) request_headers: Optional[Union[Dict[str, str], str]] = Field( None, @@ -2770,9 +2750,7 @@ class QueryProperties(BaseModel): class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] - name: str = Field( - ..., description="The stream name.", example=["Users"], title="Name" - ) + name: str = Field(..., description="The stream name.", example=["Users"], title="Name") full_refresh_stream: DeclarativeStream = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", @@ -2861,9 +2839,7 @@ class AsyncRetriever(BaseModel): ) download_extractor: Optional[ Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] - ] = Field( - None, description="Responsible for fetching the records from provided urls." - ) + ] = Field(None, description="Responsible for fetching the records from provided urls.") creation_requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", diff --git a/unit_tests/manifest_migrations/conftest.py b/unit_tests/manifest_migrations/conftest.py index 33bc5317f..638d5dc9a 100644 --- a/unit_tests/manifest_migrations/conftest.py +++ b/unit_tests/manifest_migrations/conftest.py @@ -741,7 +741,7 @@ def manifest_with_request_body_json_and_data_to_migrate_to_request_body() -> Dic # this example holds the GraphQL query object. "request_body_json": { "query": "query { {{ config['query_field'] }} { {{ config['query_value'] }} }}", - "variables": {"arg1": "test"} + "variables": {"arg1": "test"}, }, }, "record_selector": { From f30770bccbba0162c0cfa8716f7c7ee5bad25eda Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 30 Jun 2025 17:36:17 -0700 Subject: [PATCH 5/7] add default value --- .../sources/declarative/declarative_component_schema.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index ac00187cc..ce27c02f4 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4328,6 +4328,7 @@ definitions: query: type: string description: The GraphQL query to be executed + default: "query {\n \n}" additionalProperties: true DpathValidator: title: Dpath Validator From e0391b96fa91353be6c48fc339f468581826df8f Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 30 Jun 2025 17:37:55 -0700 Subject: [PATCH 6/7] remove examples from request_body --- .../declarative_component_schema.yaml | 24 ---------------- .../models/declarative_component_schema.py | 28 ------------------- 2 files changed, 52 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index ce27c02f4..e1a2532b0 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2140,30 +2140,6 @@ definitions: - stream_interval - stream_partition - stream_slice - examples: - - type: RequestBodyJsonObject - value: - sort_order: "ASC" - sort_field: "CREATED_AT" - - type: RequestBodyJsonObject - value: - key: "{{ config['value'] }}" - - type: RequestBodyJsonObject - value: - sort: - field: "updated_at" - order: "ascending" - - type: RequestBodyPlainText - value: "plain_text_body" - - type: RequestBodyUrlEncodedForm - value: - param1: "value1" - param2: "{{ config['param2_value'] }}" - - type: RequestBodyGraphQL - value: - query: - param1: "value1" - param2: "{{ config['param2_value'] }}" error_handler: title: Error Handler description: Error handler component that defines how to handle errors. diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 71a517774..f0f2d5966 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -2595,34 +2595,6 @@ class HttpRequester(BaseModelWithDeprecations): ] = Field( None, description="Specifies how to populate the body of the request with a payload. Can contain nested objects.", - examples=[ - { - "type": "RequestBodyJsonObject", - "value": {"sort_order": "ASC", "sort_field": "CREATED_AT"}, - }, - { - "type": "RequestBodyJsonObject", - "value": {"key": "{{ config['value'] }}"}, - }, - { - "type": "RequestBodyJsonObject", - "value": {"sort": {"field": "updated_at", "order": "ascending"}}, - }, - {"type": "RequestBodyPlainText", "value": "plain_text_body"}, - { - "type": "RequestBodyUrlEncodedForm", - "value": {"param1": "value1", "param2": "{{ config['param2_value'] }}"}, - }, - { - "type": "RequestBodyGraphQL", - "value": { - "query": { - "param1": "value1", - "param2": "{{ config['param2_value'] }}", - } - }, - }, - ], title="Request Body", ) error_handler: Optional[ From 9ff5e2609c640c28b23b7eafabc117d827b60c1a Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Mon, 30 Jun 2025 17:46:09 -0700 Subject: [PATCH 7/7] Update airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../request_options/interpolated_request_options_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py b/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py index ade3b99ac..cc961fae7 100644 --- a/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +++ b/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py @@ -100,7 +100,7 @@ def _resolve_request_body(self) -> None: if self.request_body.type == "RequestBodyUrlEncodedForm": self.request_body_data = self.request_body.value elif self.request_body.type == "RequestBodyGraphQL": - self.request_body_json = self.request_body.value.dict() + self.request_body_json = self.request_body.value.dict(exclude_none=True) elif self.request_body.type in ("RequestBodyJsonObject", "RequestBodyPlainText"): self.request_body_json = self.request_body.value else: