Skip to content

Commit e1da9a4

Browse files
style: fix ruff formatting
Co-Authored-By: Daryna Ishchenko <darina.ishchenko17@gmail.com>
1 parent bd7c426 commit e1da9a4

2 files changed

Lines changed: 66 additions & 88 deletions

File tree

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

Lines changed: 58 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -928,28 +928,24 @@ class OAuthConfigSpecification(BaseModel):
928928
class Config:
929929
extra = Extra.allow
930930

931-
oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = (
932-
Field(
933-
None,
934-
description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }",
935-
examples=[
936-
{"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
937-
{
938-
"app_id": {
939-
"type": "string",
940-
"path_in_connector_config": ["info", "app_id"],
941-
}
942-
},
943-
],
944-
title="OAuth user input",
945-
)
931+
oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field(
932+
None,
933+
description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }",
934+
examples=[
935+
{"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
936+
{
937+
"app_id": {
938+
"type": "string",
939+
"path_in_connector_config": ["info", "app_id"],
940+
}
941+
},
942+
],
943+
title="OAuth user input",
946944
)
947-
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = (
948-
Field(
949-
None,
950-
description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
951-
title="DeclarativeOAuth Connector Specification",
952-
)
945+
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
946+
None,
947+
description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
948+
title="DeclarativeOAuth Connector Specification",
953949
)
954950
complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
955951
None,
@@ -967,9 +963,7 @@ class Config:
967963
complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field(
968964
None,
969965
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }",
970-
examples=[
971-
{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}
972-
],
966+
examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}],
973967
title="OAuth input specification",
974968
)
975969
complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field(
@@ -1473,9 +1467,7 @@ class CustomConfigTransformation(BaseModel):
14731467
class_name: str = Field(
14741468
...,
14751469
description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.",
1476-
examples=[
1477-
"source_declarative_manifest.components.MyCustomConfigTransformation"
1478-
],
1470+
examples=["source_declarative_manifest.components.MyCustomConfigTransformation"],
14791471
)
14801472
parameters: Optional[Dict[str, Any]] = Field(
14811473
None,
@@ -1903,9 +1895,7 @@ class OAuthAuthenticator(BaseModel):
19031895
scopes: Optional[List[str]] = Field(
19041896
None,
19051897
description="List of scopes that should be granted to the access token.",
1906-
examples=[
1907-
["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]
1908-
],
1898+
examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
19091899
title="Scopes",
19101900
)
19111901
token_expiry_date: Optional[str] = Field(
@@ -2132,9 +2122,7 @@ class RecordSelector(BaseModel):
21322122
description="Responsible for filtering records to be emitted by the Source.",
21332123
title="Record Filter",
21342124
)
2135-
schema_normalization: Optional[
2136-
Union[SchemaNormalization, CustomSchemaNormalization]
2137-
] = Field(
2125+
schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
21382126
None,
21392127
description="Responsible for normalization according to the schema.",
21402128
title="Schema Normalization",
@@ -2176,12 +2164,10 @@ class DpathValidator(BaseModel):
21762164
],
21772165
title="Field Path",
21782166
)
2179-
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = (
2180-
Field(
2181-
...,
2182-
description="The condition that the specified config value will be evaluated against",
2183-
title="Validation Strategy",
2184-
)
2167+
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field(
2168+
...,
2169+
description="The condition that the specified config value will be evaluated against",
2170+
title="Validation Strategy",
21852171
)
21862172

21872173

@@ -2198,12 +2184,10 @@ class PredicateValidator(BaseModel):
21982184
],
21992185
title="Value",
22002186
)
2201-
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = (
2202-
Field(
2203-
...,
2204-
description="The validation strategy to apply to the value.",
2205-
title="Validation Strategy",
2206-
)
2187+
validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field(
2188+
...,
2189+
description="The validation strategy to apply to the value.",
2190+
title="Validation Strategy",
22072191
)
22082192

22092193

@@ -2228,12 +2212,12 @@ class ConfigAddFields(BaseModel):
22282212

22292213
class CompositeErrorHandler(BaseModel):
22302214
type: Literal["CompositeErrorHandler"]
2231-
error_handlers: List[
2232-
Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]
2233-
] = Field(
2234-
...,
2235-
description="List of error handlers to iterate on to determine how to handle a failed response.",
2236-
title="Error Handlers",
2215+
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = (
2216+
Field(
2217+
...,
2218+
description="List of error handlers to iterate on to determine how to handle a failed response.",
2219+
title="Error Handlers",
2220+
)
22372221
)
22382222
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
22392223

@@ -2395,9 +2379,9 @@ class Config:
23952379

23962380
type: Literal["DeclarativeSource"]
23972381
check: Union[CheckStream, CheckDynamicStream]
2398-
streams: Optional[
2399-
List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
2400-
] = None
2382+
streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = (
2383+
None
2384+
)
24012385
dynamic_streams: List[DynamicDeclarativeStream]
24022386
version: str = Field(
24032387
...,
@@ -2522,20 +2506,16 @@ class Config:
25222506
extra = Extra.allow
25232507

25242508
type: Literal["DeclarativeStream"]
2525-
name: Optional[str] = Field(
2526-
"", description="The stream name.", example=["Users"], title="Name"
2527-
)
2509+
name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
25282510
retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field(
25292511
...,
25302512
description="Component used to coordinate how records are extracted across stream slices and request pages.",
25312513
title="Retriever",
25322514
)
2533-
incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = (
2534-
Field(
2535-
None,
2536-
description="Component used to fetch data incrementally based on a time field in the data.",
2537-
title="Incremental Sync",
2538-
)
2515+
incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field(
2516+
None,
2517+
description="Component used to fetch data incrementally based on a time field in the data.",
2518+
title="Incremental Sync",
25392519
)
25402520
primary_key: Optional[PrimaryKey] = Field("", title="Primary Key")
25412521
schema_loader: Optional[
@@ -2709,20 +2689,18 @@ class HttpRequester(BaseModelWithDeprecations):
27092689
description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.",
27102690
title="Query Properties",
27112691
)
2712-
request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = (
2713-
Field(
2714-
None,
2715-
description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
2716-
examples=[
2717-
{"unit": "day"},
2718-
{
2719-
"query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
2720-
},
2721-
{"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
2722-
{"sort_by[asc]": "updated_at"},
2723-
],
2724-
title="Query Parameters",
2725-
)
2692+
request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field(
2693+
None,
2694+
description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
2695+
examples=[
2696+
{"unit": "day"},
2697+
{
2698+
"query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
2699+
},
2700+
{"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
2701+
{"sort_by[asc]": "updated_at"},
2702+
],
2703+
title="Query Parameters",
27262704
)
27272705
request_headers: Optional[Union[Dict[str, str], str]] = Field(
27282706
None,
@@ -2894,9 +2872,7 @@ class QueryProperties(BaseModel):
28942872

28952873
class StateDelegatingStream(BaseModel):
28962874
type: Literal["StateDelegatingStream"]
2897-
name: str = Field(
2898-
..., description="The stream name.", example=["Users"], title="Name"
2899-
)
2875+
name: str = Field(..., description="The stream name.", example=["Users"], title="Name")
29002876
full_refresh_stream: DeclarativeStream = Field(
29012877
...,
29022878
description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.",
@@ -2983,17 +2959,13 @@ class AsyncRetriever(BaseModel):
29832959
status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field(
29842960
..., description="Responsible for fetching the actual status of the async job."
29852961
)
2986-
download_target_extractor: Optional[
2987-
Union[DpathExtractor, CustomRecordExtractor]
2988-
] = Field(
2962+
download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field(
29892963
None,
29902964
description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.",
29912965
)
29922966
download_extractor: Optional[
29932967
Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor]
2994-
] = Field(
2995-
None, description="Responsible for fetching the records from provided urls."
2996-
)
2968+
] = Field(None, description="Responsible for fetching the records from provided urls.")
29972969
creation_requester: Union[HttpRequester, CustomRequester] = Field(
29982970
...,
29992971
description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.",

airbyte_cdk/sources/streams/http/http_client.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,11 @@ def __str__(self) -> str:
102102
class HttpClient:
103103
_DEFAULT_MAX_RETRY: int = 5
104104
_DEFAULT_MAX_TIME: int = 60 * 10
105-
_ACTIONS_TO_RETRY_ON = {ResponseAction.RETRY, ResponseAction.RATE_LIMITED, ResponseAction.REFRESH_TOKEN_THEN_RETRY}
105+
_ACTIONS_TO_RETRY_ON = {
106+
ResponseAction.RETRY,
107+
ResponseAction.RATE_LIMITED,
108+
ResponseAction.REFRESH_TOKEN_THEN_RETRY,
109+
}
106110

107111
def __init__(
108112
self,
@@ -466,7 +470,9 @@ def _handle_error_resolution(
466470
token, expires_in = self._session.auth.refresh_access_token() # type: ignore[union-attr]
467471
self._session.auth.access_token = token # type: ignore[union-attr]
468472
self._session.auth.set_token_expiry_date(expires_in) # type: ignore[union-attr]
469-
self._logger.info("Refreshed OAuth token due to REFRESH_TOKEN_THEN_RETRY response action")
473+
self._logger.info(
474+
"Refreshed OAuth token due to REFRESH_TOKEN_THEN_RETRY response action"
475+
)
470476
except Exception as refresh_error:
471477
self._logger.warning(
472478
f"Failed to refresh OAuth token: {refresh_error}. Proceeding with retry using existing token."

0 commit comments

Comments
 (0)