-
Notifications
You must be signed in to change notification settings - Fork 45
Expand file tree
/
Copy path__init__.py
More file actions
366 lines (343 loc) · 11.8 KB
/
__init__.py
File metadata and controls
366 lines (343 loc) · 11.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
"""
# Welcome to the Airbyte Python CDK!
The Airbyte Python CDK is a Python library that provides a set of tools to help you build
connectors for the Airbyte platform.
## Building Source Connectors
To build a source connector, you will want to refer to
the following classes and modules:
- `airbyte_cdk.sources`
- `airbyte_cdk.sources.concurrent_source`
- `airbyte_cdk.sources.config`
- `airbyte_cdk.sources.file_based`
- `airbyte_cdk.sources.streams`
## Building Destination Connectors
To build a destination connector, you will want to refer to
the following classes and modules:
- `airbyte_cdk.destinations`
- `airbyte_cdk.destinations.Destination`
- `airbyte_cdk.destinations.vector_db_based`
## Working with Airbyte Protocol Models
The Airbyte CDK provides a set of classes that help you work with the Airbyte protocol models:
- `airbyte_cdk.models.airbyte_protocol`
- `airbyte_cdk.models.airbyte_protocol_serializers`
## Using the CLI (`airbyte_cdk.cli`)
The Airbyte CDK provides two command-line interfaces (CLIs) for interacting with the framework.
- `airbyte-cdk`: This is the main CLI for the Airbyte CDK. It provides commands for building
and testing connectors, as well as other utilities. See the `airbyte_cdk.cli.airbyte_cdk` module
for more details.
- `source-declarative-manifest`: This command allows you to run declarative manifests directly.
See the `airbyte_cdk.cli.source_declarative_manifest` module for more details.
---
API Reference
---
"""
# Warning: The below imports are not stable and will cause circular
# dependencies if auto-sorted with isort. Please keep them in the same order.
# TODO: Submodules should import from lower-level modules, rather than importing from here.
# Imports should also be placed in `if TYPE_CHECKING` blocks if they are only used as type
# hints - again, to avoid circular dependencies.
# Once those issues are resolved, the below can be sorted with isort.
import dunamai as _dunamai
from .config_observation import (
create_connector_config_control_message,
emit_configuration_as_airbyte_control_message,
)
from .connector import BaseConnector, Connector
from .destinations import Destination
from .entrypoint import AirbyteEntrypoint, launch
from .logger import AirbyteLogFormatter, init_logger
from .models import (
AdvancedAuth,
AirbyteConnectionStatus,
AirbyteLogMessage,
AirbyteMessage,
AirbyteRecordMessage,
AirbyteStream,
ConfiguredAirbyteCatalog,
ConfiguredAirbyteStream,
ConnectorSpecification,
DestinationSyncMode,
FailureType,
Level,
OAuthConfigSpecification,
OrchestratorType,
Status,
SyncMode,
Type,
)
from .sources import AbstractSource, Source
from .sources.concurrent_source.concurrent_source import ConcurrentSource
from .sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
from .sources.config import BaseConfig
from .sources.connector_state_manager import ConnectorStateManager
from .sources.declarative.auth import DeclarativeOauth2Authenticator
from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth
from .sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
from .sources.declarative.auth.token import (
ApiKeyAuthenticator,
BasicHttpAuthenticator,
BearerAuthenticator,
)
from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
from .sources.declarative.declarative_stream import DeclarativeStream
from .sources.declarative.decoders import Decoder, JsonDecoder
from .sources.declarative.exceptions import ReadException
from .sources.declarative.extractors import DpathExtractor, RecordSelector
from .sources.declarative.extractors.record_extractor import RecordExtractor
from .sources.declarative.extractors.record_filter import RecordFilter
from .sources.declarative.incremental import DatetimeBasedCursor
from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
LegacyToPerPartitionStateMigration,
)
from .sources.declarative.partition_routers import (
CartesianProductStreamSlicer,
SinglePartitionRouter,
SubstreamPartitionRouter,
)
from .sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
from .sources.declarative.requesters import HttpRequester, Requester
from .sources.declarative.requesters.error_handlers import BackoffStrategy
from .sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
from .sources.declarative.requesters.paginators.strategies import (
CursorPaginationStrategy,
OffsetIncrement,
PageIncrement,
StopConditionPaginationStrategyDecorator,
)
from .sources.declarative.requesters.request_option import RequestOption, RequestOptionType
from .sources.declarative.requesters.request_options.default_request_options_provider import (
DefaultRequestOptionsProvider,
)
from .sources.declarative.requesters.request_options.interpolated_request_input_provider import (
InterpolatedRequestInputProvider,
)
from .sources.declarative.requesters.requester import HttpMethod
from .sources.declarative.retrievers import SimpleRetriever
from .sources.declarative.schema import JsonFileSchemaLoader
from .sources.declarative.transformations.add_fields import AddedFieldDefinition, AddFields
from .sources.declarative.transformations.transformation import RecordTransformation
from .sources.declarative.types import FieldPointer
from .sources.declarative.yaml_declarative_source import YamlDeclarativeSource
from .sources.message import InMemoryMessageRepository, MessageRepository
from .sources.source import TState
from .sources.streams.availability_strategy import AvailabilityStrategy
from .sources.streams.call_rate import (
AbstractAPIBudget,
CachedLimiterSession,
HttpAPIBudget,
HttpRequestMatcher,
LimiterSession,
MovingWindowCallRatePolicy,
Rate,
)
from .sources.streams.checkpoint import Cursor as LegacyCursor
from .sources.streams.checkpoint import ResumableFullRefreshCursor
from .sources.streams.concurrent.adapters import StreamFacade
from .sources.streams.concurrent.cursor import (
ConcurrentCursor,
Cursor,
CursorField,
FinalStateCursor,
)
from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
EpochValueConcurrentStreamStateConverter,
IsoMillisConcurrentStreamStateConverter,
)
from .sources.streams.core import IncrementalMixin, Stream, package_name_from_class
from .sources.streams.http import HttpStream, HttpSubStream
from .sources.streams.http.availability_strategy import HttpAvailabilityStrategy
from .sources.streams.http.exceptions import (
BaseBackoffException,
DefaultBackoffException,
UserDefinedBackoffException,
)
from .sources.streams.http.rate_limiting import default_backoff_handler
from .sources.streams.http.requests_native_auth import (
Oauth2Authenticator,
SingleUseRefreshTokenOauth2Authenticator,
TokenAuthenticator,
)
from .sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
from .sources.types import Config, Record, StreamSlice
from .sources.utils import casing
from .sources.utils.schema_helpers import (
InternalConfig,
ResourceSchemaLoader,
check_config_against_spec_or_exit,
expand_refs,
split_config,
)
from .sources.utils.transform import TransformConfig, TypeTransformer
from .utils import AirbyteTracedException, is_cloud_environment
from .utils.constants import ENV_REQUEST_CACHE_PATH
from .utils.event_timing import create_timer
from .utils.oneof_option_config import OneOfOptionConfig
from .utils.spec_schema_transformations import resolve_refs
from .utils.stream_status_utils import as_airbyte_message
__all__ = [
# Availability strategy
"AvailabilityStrategy",
"HttpAvailabilityStrategy",
# Checkpoint
"LegacyCursor",
"ResumableFullRefreshCursor",
# Concurrent
"ConcurrentCursor",
"ConcurrentSource",
"ConcurrentSourceAdapter",
"Cursor",
"CursorField",
"DEFAULT_CONCURRENCY",
"EpochValueConcurrentStreamStateConverter",
"FinalStateCursor",
"IsoMillisConcurrentStreamStateConverter",
"StreamFacade",
# Config observation
"create_connector_config_control_message",
"emit_configuration_as_airbyte_control_message",
# Connector
"AbstractSource",
"BaseConfig",
"BaseConnector",
"Connector",
"Destination",
"Source",
"TState",
# Declarative
"AddFields",
"AddedFieldDefinition",
"ApiKeyAuthenticator",
"BackoffStrategy",
"BasicHttpAuthenticator",
"BearerAuthenticator",
"CartesianProductStreamSlicer",
"CursorPaginationStrategy",
"DatetimeBasedCursor",
"DeclarativeAuthenticator",
"DeclarativeOauth2Authenticator",
"DeclarativeSingleUseRefreshTokenOauth2Authenticator",
"DeclarativeStream",
"Decoder",
"DefaultPaginator",
"DefaultRequestOptionsProvider",
"DpathExtractor",
"FieldPointer",
"HttpMethod",
"HttpRequester",
"InterpolatedBoolean",
"InterpolatedRequestInputProvider",
"InterpolatedString",
"JsonDecoder",
"JsonFileSchemaLoader",
"LegacyToPerPartitionStateMigration",
"MinMaxDatetime",
"NoAuth",
"OffsetIncrement",
"PageIncrement",
"PaginationStrategy",
"ParentStreamConfig",
"ReadException",
"RecordExtractor",
"RecordFilter",
"RecordSelector",
"RecordTransformation",
"RequestOption",
"RequestOptionType",
"Requester",
"ResponseStatus",
"SimpleRetriever",
"SinglePartitionRouter",
"StopConditionPaginationStrategyDecorator",
"StreamSlice",
"SubstreamPartitionRouter",
"YamlDeclarativeSource",
# Entrypoint
"launch",
"AirbyteEntrypoint",
# HTTP
"AbstractAPIBudget",
"AbstractHeaderAuthenticator",
"BaseBackoffException",
"CachedLimiterSession",
"DefaultBackoffException",
"default_backoff_handler",
"HttpAPIBudget",
"HttpAuthenticator",
"HttpRequestMatcher",
"HttpStream",
"HttpSubStream",
"LimiterSession",
"MovingWindowCallRatePolicy",
"MultipleTokenAuthenticator",
"Oauth2Authenticator",
"Rate",
"SingleUseRefreshTokenOauth2Authenticator",
"TokenAuthenticator",
"UserDefinedBackoffException",
# Logger
"AirbyteLogFormatter",
"init_logger",
# Protocol classes
"AirbyteStream",
"AirbyteConnectionStatus",
"AirbyteMessage",
"ConfiguredAirbyteCatalog",
"Status",
"Type",
"OrchestratorType",
"ConfiguredAirbyteStream",
"DestinationSyncMode",
"SyncMode",
"FailureType",
"AdvancedAuth",
"AirbyteLogMessage",
"OAuthConfigSpecification",
"ConnectorSpecification",
"Level",
"AirbyteRecordMessage",
# Repository
"InMemoryMessageRepository",
"MessageRepository",
# State management
"ConnectorStateManager",
# Stream
"IncrementalMixin",
"Stream",
"StreamData",
"package_name_from_class",
# Utils
"AirbyteTracedException",
"is_cloud_environment",
"casing",
"InternalConfig",
"ResourceSchemaLoader",
"check_config_against_spec_or_exit",
"split_config",
"TransformConfig",
"TypeTransformer",
"ENV_REQUEST_CACHE_PATH",
"create_timer",
"OneOfOptionConfig",
"resolve_refs",
"as_airbyte_message",
# Types
"Config",
"Record",
"Source",
"StreamSlice",
]
__version__: str
"""Version generated by poetry dynamic versioning during publish.
When running in development, dunamai will calculate a new prerelease version
from existing git release tag info.
"""
try:
__version__ = _dunamai.get_version(
"airbyte-cdk",
third_choice=_dunamai.Version.from_any_vcs,
fallback=_dunamai.Version("0.0.0+dev"),
).serialize()
except:
__version__ = "0.0.0+dev"