Skip to content

Commit db8312d

Browse files
author
maxime.c
committed
only apply concurrency changes on the test reads
1 parent 3205424 commit db8312d

2 files changed

Lines changed: 13 additions & 10 deletions

File tree

airbyte_cdk/manifest_server/command_processor/utils.py

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -64,16 +64,7 @@ def build_source(
6464
page_limit: Optional[int] = None,
6565
slice_limit: Optional[int] = None,
6666
) -> ConcurrentDeclarativeSource:
67-
# We enforce a concurrency level of 1 so that the stream is processed on a single thread
68-
# to retain ordering for the grouping of the builder message responses.
6967
definition = copy.deepcopy(manifest)
70-
if "concurrency_level" in definition:
71-
definition["concurrency_level"]["default_concurrency"] = 1
72-
else:
73-
definition["concurrency_level"] = {
74-
"type": "ConcurrencyLevel",
75-
"default_concurrency": 2,
76-
}
7768

7869
should_normalize = should_normalize_manifest(manifest)
7970
if should_normalize:

airbyte_cdk/manifest_server/routers/manifest.py

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import copy
12
import hashlib
23
from dataclasses import asdict
34
from typing import Any, Dict, List, Mapping, Optional
@@ -87,8 +88,19 @@ def test_read(request: StreamTestReadRequest) -> StreamReadResponse:
8788
"md5": hashlib.md5(request.custom_components_code.encode()).hexdigest()
8889
}
8990

91+
# We enforce a concurrency level of 1 so that the stream is processed on a single thread
92+
# to retain ordering for the grouping of the builder message responses.
93+
manifest = copy.deepcopy(request.manifest.model_dump())
94+
if "concurrency_level" in manifest:
95+
manifest["concurrency_level"]["default_concurrency"] = 1
96+
else:
97+
manifest["concurrency_level"] = {
98+
"type": "ConcurrencyLevel",
99+
"default_concurrency": 1,
100+
}
101+
90102
source = safe_build_source(
91-
request.manifest.model_dump(),
103+
manifest,
92104
config_dict,
93105
catalog,
94106
converted_state,

0 commit comments

Comments
 (0)