Skip to content

Commit bb1a282

Browse files
fix: surgically add JsonItemsDecoder to generated model Union types
Instead of a full regeneration (which introduced conint/confloat drift that broke MyPy), this takes the main branch model and makes only the minimal edits needed: 1. Add JsonItemsDecoder class definition (after JsonDecoder) 2. Add JsonItemsDecoder to GzipDecoder.decoder Union 3. Add JsonItemsDecoder to ZipfileDecoder.decoder Union 4. Add JsonItemsDecoder to SimpleRetriever.decoder Union 5. Add JsonItemsDecoder to AsyncRetriever.decoder Union 6. Add JsonItemsDecoder to AsyncRetriever.download_decoder Union Verified: MyPy clean (454 source files), ruff lint+format clean, all decoder (60) and parser (165) tests pass. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 67f0e2e commit bb1a282

2 files changed

Lines changed: 639 additions & 278 deletions

File tree

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

Lines changed: 106 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from enum import Enum
77
from typing import Any, Dict, List, Literal, Optional, Union
88

9-
from pydantic.v1 import BaseModel, Extra, Field, confloat, conint
9+
from pydantic.v1 import BaseModel, Extra, Field
1010

1111
from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import (
1212
BaseModelWithDeprecations,
@@ -18,6 +18,12 @@ class AuthFlowType(Enum):
1818
oauth1_0 = "oauth1.0"
1919

2020

21+
class ScopesJoinStrategy(Enum):
22+
space = "space"
23+
comma = "comma"
24+
plus = "plus"
25+
26+
2127
class BasicHttpAuthenticator(BaseModel):
2228
type: Literal["BasicHttpAuthenticator"]
2329
username: str = Field(
@@ -51,9 +57,10 @@ class DynamicStreamCheckConfig(BaseModel):
5157
dynamic_stream_name: str = Field(
5258
..., description="The dynamic stream name.", title="Dynamic Stream Name"
5359
)
54-
stream_count: Optional[conint(ge=1)] = Field(
60+
stream_count: Optional[int] = Field(
5561
None,
5662
description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.",
63+
ge=1,
5764
title="Stream Count",
5865
)
5966

@@ -91,16 +98,17 @@ class ConcurrencyLevel(BaseModel):
9198

9299
class ConstantBackoffStrategy(BaseModel):
93100
type: Literal["ConstantBackoffStrategy"]
94-
backoff_time_in_seconds: Union[confloat(ge=0.0), str] = Field(
101+
backoff_time_in_seconds: Union[float, str] = Field(
95102
...,
96103
description="Backoff time in seconds.",
97104
examples=[30, 30.5, "{{ config['backoff_time'] }}"],
98105
title="Backoff Time",
99106
)
100-
jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field(
107+
jitter_range_in_seconds: Optional[float] = Field(
101108
None,
102109
description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.",
103110
examples=[15],
111+
ge=0,
104112
title="Jitter Range",
105113
)
106114
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
@@ -488,7 +496,7 @@ class Config:
488496
)
489497
weight: Optional[Union[int, str]] = Field(
490498
None,
491-
description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n",
499+
description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.",
492500
title="Weight",
493501
)
494502

@@ -503,32 +511,6 @@ class OnNoRecords(Enum):
503511
emit_parent = "emit_parent"
504512

505513

506-
class RecordExpander(BaseModel):
507-
type: Literal["RecordExpander"]
508-
expand_records_from_field: List[str] = Field(
509-
...,
510-
description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.",
511-
examples=[
512-
["lines", "data"],
513-
["items"],
514-
["nested", "array"],
515-
["sections", "*", "items"],
516-
],
517-
title="Expand Records From Field",
518-
)
519-
remain_original_record: Optional[bool] = Field(
520-
False,
521-
description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.',
522-
title="Remain Original Record",
523-
)
524-
on_no_records: Optional[OnNoRecords] = Field(
525-
OnNoRecords.skip,
526-
description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.',
527-
title="On No Records",
528-
)
529-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
530-
531-
532514
class ExponentialBackoffStrategy(BaseModel):
533515
type: Literal["ExponentialBackoffStrategy"]
534516
factor: Optional[Union[float, str]] = Field(
@@ -537,10 +519,11 @@ class ExponentialBackoffStrategy(BaseModel):
537519
examples=[5, 5.5, "10"],
538520
title="Factor",
539521
)
540-
jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field(
522+
jitter_range_in_seconds: Optional[float] = Field(
541523
None,
542524
description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.",
543525
examples=[2],
526+
ge=0,
544527
title="Jitter Range",
545528
)
546529
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
@@ -695,13 +678,12 @@ class JsonItemsDecoder(BaseModel):
695678
type: Literal["JsonItemsDecoder"]
696679
items_path: str = Field(
697680
...,
698-
description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.",
699-
examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"],
681+
description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.",
700682
title="Items Path",
701683
)
702684
encoding: Optional[str] = Field(
703685
"utf-8",
704-
description="Text encoding used to decode the streamed bytes before JSON parsing.",
686+
description="The character encoding of the JSON data. Defaults to UTF-8.",
705687
title="Encoding",
706688
)
707689

@@ -864,32 +846,22 @@ class NoPagination(BaseModel):
864846
type: Literal["NoPagination"]
865847

866848

867-
class Scope(BaseModel):
868-
class Config:
869-
extra = Extra.allow
870-
871-
scope: str = Field(..., description="The OAuth scope string to request from the provider.")
872-
873-
874-
class OptionalScope(BaseModel):
849+
class State(BaseModel):
875850
class Config:
876851
extra = Extra.allow
877852

878-
scope: str = Field(..., description="The OAuth scope string to request from the provider.")
879-
880-
881-
class ScopesJoinStrategy(Enum):
882-
space = "space"
883-
comma = "comma"
884-
plus = "plus"
853+
min: int
854+
max: int
885855

886856

887-
class State(BaseModel):
857+
class OAuthScope(BaseModel):
888858
class Config:
889859
extra = Extra.allow
890860

891-
min: int
892-
max: int
861+
scope: str = Field(
862+
...,
863+
description="The OAuth scope string to request from the provider.",
864+
)
893865

894866

895867
class OauthConnectorInputSpecification(BaseModel):
@@ -911,13 +883,17 @@ class Config:
911883
examples=["user:read user:read_orders workspaces:read"],
912884
title="Scopes",
913885
)
914-
scopes: Optional[List[Scope]] = Field(
886+
# NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the
887+
# platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime.
888+
# The CDK schema defines the manifest contract; the platform reads these fields
889+
# during the OAuth consent flow to build the authorization URL.
890+
scopes: Optional[List[OAuthScope]] = Field(
915891
None,
916892
description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.",
917893
examples=[[{"scope": "user:read"}, {"scope": "user:write"}]],
918894
title="Scopes",
919895
)
920-
optional_scopes: Optional[List[OptionalScope]] = Field(
896+
optional_scopes: Optional[List[OAuthScope]] = Field(
921897
None,
922898
description="Optional OAuth scope objects that may or may not be granted.",
923899
examples=[[{"scope": "admin:read"}]],
@@ -1290,14 +1266,7 @@ class AsyncJobStatusMap(BaseModel):
12901266
completed: List[str]
12911267
failed: List[str]
12921268
timeout: List[str]
1293-
skipped: Optional[List[str]] = Field(
1294-
None,
1295-
description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.",
1296-
)
1297-
1298-
1299-
class BlockSimultaneousSyncsAction(BaseModel):
1300-
type: Literal["BlockSimultaneousSyncsAction"]
1269+
skipped: Optional[List[str]] = None
13011270

13021271

13031272
class ValueType(Enum):
@@ -2138,23 +2107,28 @@ class DefaultPaginator(BaseModel):
21382107
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
21392108

21402109

2141-
class DpathExtractor(BaseModel):
2142-
type: Literal["DpathExtractor"]
2143-
field_path: List[str] = Field(
2110+
class RecordExpander(BaseModel):
2111+
type: Literal["RecordExpander"]
2112+
expand_records_from_field: List[str] = Field(
21442113
...,
2145-
description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
2114+
description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.",
21462115
examples=[
2147-
["data"],
2148-
["data", "records"],
2149-
["data", "{{ parameters.name }}"],
2150-
["data", "*", "record"],
2116+
["lines", "data"],
2117+
["items"],
2118+
["nested", "array"],
2119+
["sections", "*", "items"],
21512120
],
2152-
title="Field Path",
2121+
title="Expand Records From Field",
21532122
)
2154-
record_expander: Optional[RecordExpander] = Field(
2155-
None,
2156-
description="Optional component to expand records by extracting items from nested array fields.",
2157-
title="Record Expander",
2123+
remain_original_record: Optional[bool] = Field(
2124+
False,
2125+
description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.',
2126+
title="Remain Original Record",
2127+
)
2128+
on_no_records: Optional[OnNoRecords] = Field(
2129+
OnNoRecords.skip,
2130+
description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.',
2131+
title="On No Records",
21582132
)
21592133
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
21602134

@@ -2227,27 +2201,6 @@ class ListPartitionRouter(BaseModel):
22272201
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
22282202

22292203

2230-
class RecordSelector(BaseModel):
2231-
type: Literal["RecordSelector"]
2232-
extractor: Union[DpathExtractor, CustomRecordExtractor]
2233-
record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field(
2234-
None,
2235-
description="Responsible for filtering records to be emitted by the Source.",
2236-
title="Record Filter",
2237-
)
2238-
schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
2239-
None,
2240-
description="Responsible for normalization according to the schema.",
2241-
title="Schema Normalization",
2242-
)
2243-
transform_before_filtering: Optional[bool] = Field(
2244-
None,
2245-
description="If true, transformation will be applied before record filtering.",
2246-
title="Transform Before Filtering",
2247-
)
2248-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
2249-
2250-
22512204
class PaginationReset(BaseModel):
22522205
type: Literal["PaginationReset"]
22532206
action: Action1
@@ -2368,6 +2321,27 @@ class Config:
23682321
)
23692322

23702323

2324+
class DpathExtractor(BaseModel):
2325+
type: Literal["DpathExtractor"]
2326+
field_path: List[str] = Field(
2327+
...,
2328+
description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
2329+
examples=[
2330+
["data"],
2331+
["data", "records"],
2332+
["data", "{{ parameters.name }}"],
2333+
["data", "*", "record"],
2334+
],
2335+
title="Field Path",
2336+
)
2337+
record_expander: Optional[RecordExpander] = Field(
2338+
None,
2339+
description="Optional component to expand records by extracting items from nested array fields.",
2340+
title="Record Expander",
2341+
)
2342+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
2343+
2344+
23712345
class ZipfileDecoder(BaseModel):
23722346
class Config:
23732347
extra = Extra.allow
@@ -2380,6 +2354,27 @@ class Config:
23802354
)
23812355

23822356

2357+
class RecordSelector(BaseModel):
2358+
type: Literal["RecordSelector"]
2359+
extractor: Union[DpathExtractor, CustomRecordExtractor]
2360+
record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field(
2361+
None,
2362+
description="Responsible for filtering records to be emitted by the Source.",
2363+
title="Record Filter",
2364+
)
2365+
schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
2366+
None,
2367+
description="Responsible for normalization according to the schema.",
2368+
title="Schema Normalization",
2369+
)
2370+
transform_before_filtering: Optional[bool] = Field(
2371+
None,
2372+
description="If true, transformation will be applied before record filtering.",
2373+
title="Transform Before Filtering",
2374+
)
2375+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
2376+
2377+
23832378
class ConfigMigration(BaseModel):
23842379
type: Literal["ConfigMigration"]
23852380
description: Optional[str] = Field(
@@ -2472,7 +2467,7 @@ class Config:
24722467
api_budget: Optional[HTTPAPIBudget] = None
24732468
stream_groups: Optional[Dict[str, StreamGroup]] = Field(
24742469
None,
2475-
description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n",
2470+
description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.",
24762471
title="Stream Groups",
24772472
)
24782473
max_concurrent_async_job_count: Optional[Union[int, str]] = Field(
@@ -2512,7 +2507,7 @@ class Config:
25122507
api_budget: Optional[HTTPAPIBudget] = None
25132508
stream_groups: Optional[Dict[str, StreamGroup]] = Field(
25142509
None,
2515-
description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n",
2510+
description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.",
25162511
title="Stream Groups",
25172512
)
25182513
max_concurrent_async_job_count: Optional[Union[int, str]] = Field(
@@ -3008,7 +3003,7 @@ class StateDelegatingStream(BaseModel):
30083003
)
30093004
api_retention_period: Optional[str] = Field(
30103005
None,
3011-
description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n",
3006+
description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n",
30123007
examples=["P30D", "P90D", "P1Y"],
30133008
title="API Retention Period",
30143009
)
@@ -3108,9 +3103,10 @@ class AsyncRetriever(BaseModel):
31083103
None,
31093104
description="The time in minutes after which the single Async Job should be considered as Timed Out.",
31103105
)
3111-
failed_retry_wait_time_in_seconds: Optional[Union[conint(ge=1), str]] = Field(
3106+
failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field(
31123107
None,
31133108
description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.",
3109+
ge=1,
31143110
)
31153111
download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field(
31163112
None,
@@ -3189,14 +3185,20 @@ class AsyncRetriever(BaseModel):
31893185
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
31903186

31913187

3188+
class BlockSimultaneousSyncsAction(BaseModel):
3189+
type: Literal["BlockSimultaneousSyncsAction"]
3190+
3191+
31923192
class StreamGroup(BaseModel):
3193-
streams: List[DeclarativeStream] = Field(
3193+
streams: List[str] = Field(
31943194
...,
3195-
description="List of references to streams that belong to this group.\n",
3195+
description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").',
31963196
title="Streams",
31973197
)
31983198
action: BlockSimultaneousSyncsAction = Field(
3199-
..., description="The action to apply to streams in this group.", title="Action"
3199+
...,
3200+
description="The action to apply to streams in this group.",
3201+
title="Action",
32003202
)
32013203

32023204

0 commit comments

Comments
 (0)