From f82b76ae18d32dcfb8cac6cc4595f49f18b2484f Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 11 Apr 2025 20:45:02 -0700 Subject: [PATCH 01/12] [cherry-pick-me][chore]: resolve pytest warnings undeclared marks, and for classes starting with "Test" --- airbyte_cdk/connector_builder/connector_builder_handler.py | 2 ++ airbyte_cdk/connector_builder/test_reader/reader.py | 2 ++ pytest.ini | 3 +++ .../sources/declarative/decoders/test_composite_decoder.py | 2 ++ unit_tests/sources/declarative/parsers/testing_components.py | 4 ++++ .../sources/declarative/test_yaml_declarative_source.py | 2 ++ unit_tests/sources/file_based/scenarios/scenario_builder.py | 2 ++ .../sources/file_based/test_file_based_stream_reader.py | 4 ++++ 8 files changed, 21 insertions(+) diff --git a/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte_cdk/connector_builder/connector_builder_handler.py index 27929dfa2..8bc9004fa 100644 --- a/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -37,6 +37,8 @@ @dataclass class TestLimits: + __test__: bool = False # Prevent pytest from treating this as a test case, despite its name + max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS) max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE) max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES) diff --git a/airbyte_cdk/connector_builder/test_reader/reader.py b/airbyte_cdk/connector_builder/test_reader/reader.py index b776811eb..3955d7a3b 100644 --- a/airbyte_cdk/connector_builder/test_reader/reader.py +++ b/airbyte_cdk/connector_builder/test_reader/reader.py @@ -66,6 +66,8 @@ class TestReader: """ + __test__: bool = False # Prevent pytest from treating this as a test case, despite its name + logger = logging.getLogger("airbyte.connector-builder") def __init__( diff --git a/pytest.ini b/pytest.ini index 6a5cafc1f..2aaee685b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,3 +5,6 @@ log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno log_cli_date_format=%Y-%m-%d %H:%M:%S filterwarnings = ignore::airbyte_cdk.sources.source.ExperimentalClassWarning +markers = + slow: mark tests as slow + asyncio: mark test as asyncio diff --git a/unit_tests/sources/declarative/decoders/test_composite_decoder.py b/unit_tests/sources/declarative/decoders/test_composite_decoder.py index 02c0993b6..03c947637 100644 --- a/unit_tests/sources/declarative/decoders/test_composite_decoder.py +++ b/unit_tests/sources/declarative/decoders/test_composite_decoder.py @@ -259,6 +259,8 @@ def test_composite_raw_decoder_csv_parser_values(requests_mock, encoding: str, d class TestServer(BaseHTTPRequestHandler): + __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + def do_GET(self) -> None: self.send_response(200) self.end_headers() diff --git a/unit_tests/sources/declarative/parsers/testing_components.py b/unit_tests/sources/declarative/parsers/testing_components.py index 0d49e8627..3145368a8 100644 --- a/unit_tests/sources/declarative/parsers/testing_components.py +++ b/unit_tests/sources/declarative/parsers/testing_components.py @@ -21,6 +21,8 @@ class TestingSomeComponent(DefaultErrorHandler): A basic test class with various field permutations used to test manifests with custom components """ + __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + subcomponent_field_with_hint: DpathExtractor = field( default_factory=lambda: DpathExtractor(field_path=[], config={}, parameters={}) ) @@ -37,5 +39,7 @@ class TestingCustomSubstreamPartitionRouter(SubstreamPartitionRouter): A test class based on a SubstreamPartitionRouter used for testing manifests that use custom components. """ + __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + custom_field: str custom_pagination_strategy: PaginationStrategy diff --git a/unit_tests/sources/declarative/test_yaml_declarative_source.py b/unit_tests/sources/declarative/test_yaml_declarative_source.py index 67f198eb2..c4e5f0bfd 100644 --- a/unit_tests/sources/declarative/test_yaml_declarative_source.py +++ b/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -133,6 +133,8 @@ def test_source_with_missing_reference_fails(self): class TestFileContent: + __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + def __init__(self, content): self.file = tempfile.NamedTemporaryFile(mode="w", delete=False) diff --git a/unit_tests/sources/file_based/scenarios/scenario_builder.py b/unit_tests/sources/file_based/scenarios/scenario_builder.py index da8c7ba87..f919622e8 100644 --- a/unit_tests/sources/file_based/scenarios/scenario_builder.py +++ b/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -42,6 +42,8 @@ def build( class TestScenario(Generic[SourceType]): + __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + def __init__( self, name: str, diff --git a/unit_tests/sources/file_based/test_file_based_stream_reader.py b/unit_tests/sources/file_based/test_file_based_stream_reader.py index 4a9d4e349..366599505 100644 --- a/unit_tests/sources/file_based/test_file_based_stream_reader.py +++ b/unit_tests/sources/file_based/test_file_based_stream_reader.py @@ -62,6 +62,8 @@ class TestStreamReader(AbstractFileBasedStreamReader): + __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + @property def config(self) -> Optional[AbstractFileBasedSpec]: return self._config @@ -100,6 +102,8 @@ def identities_schema(self) -> Dict[str, Any]: class TestSpec(AbstractFileBasedSpec): + __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + @classmethod def documentation_url(cls) -> AnyUrl: return AnyUrl(scheme="https", url="https://docs.airbyte.com/integrations/sources/test") # type: ignore From f091b53cfb566a04794e508b64e1ad879e8b3170 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 11 Apr 2025 21:04:08 -0700 Subject: [PATCH 02/12] [cherry-pick-me][fix]: dataclasses constructor break with __test__ member --- airbyte_cdk/connector_builder/connector_builder_handler.py | 2 +- unit_tests/sources/declarative/parsers/testing_components.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte_cdk/connector_builder/connector_builder_handler.py index 8bc9004fa..8beb19f34 100644 --- a/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -35,7 +35,7 @@ MAX_STREAMS_KEY = "max_streams" -@dataclass +@dataclass(kw_only=True) class TestLimits: __test__: bool = False # Prevent pytest from treating this as a test case, despite its name diff --git a/unit_tests/sources/declarative/parsers/testing_components.py b/unit_tests/sources/declarative/parsers/testing_components.py index 3145368a8..545bf1e2f 100644 --- a/unit_tests/sources/declarative/parsers/testing_components.py +++ b/unit_tests/sources/declarative/parsers/testing_components.py @@ -15,7 +15,7 @@ ) -@dataclass +@dataclass(kw_only=True) class TestingSomeComponent(DefaultErrorHandler): """ A basic test class with various field permutations used to test manifests with custom components @@ -33,7 +33,7 @@ class TestingSomeComponent(DefaultErrorHandler): paginator: DefaultPaginator = None -@dataclass +@dataclass(kw_only=True) class TestingCustomSubstreamPartitionRouter(SubstreamPartitionRouter): """ A test class based on a SubstreamPartitionRouter used for testing manifests that use custom components. From 12afc3d53232790a38a44aa684613070092e2771 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 11 Apr 2025 21:09:39 -0700 Subject: [PATCH 03/12] [cherry-pick-me]: use kw args --- airbyte_cdk/connector_builder/connector_builder_handler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte_cdk/connector_builder/connector_builder_handler.py index 8beb19f34..220188d34 100644 --- a/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -53,7 +53,12 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits: max_slices = command_config.get(MAX_SLICES_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_SLICES max_records = command_config.get(MAX_RECORDS_KEY) or DEFAULT_MAXIMUM_RECORDS max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS - return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams) + return TestLimits( + max_records=max_records, + max_pages_per_slice=max_pages_per_slice, + max_slices=max_slices, + max_streams=max_streams, + ) def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource: From cca55f798e70626f6bc1f5bab05eb7a505191333 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 11 Apr 2025 21:13:23 -0700 Subject: [PATCH 04/12] [cherry-pick-me]: kw args --- .../connector_builder/test_connector_builder_handler.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/unit_tests/connector_builder/test_connector_builder_handler.py b/unit_tests/connector_builder/test_connector_builder_handler.py index 5c537811b..e65c2d69f 100644 --- a/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/unit_tests/connector_builder/test_connector_builder_handler.py @@ -979,11 +979,15 @@ def test_get_limits( assert limits.max_slices == expected_max_slices -def test_create_source(): +def test_create_source() -> None: max_records = 3 max_pages_per_slice = 2 max_slices = 1 - limits = TestLimits(max_records, max_pages_per_slice, max_slices) + limits = TestLimits( + max_records=max_records, + max_pages_per_slice=max_pages_per_slice, + max_slices=max_slices, + ) config = {"__injected_declarative_manifest": MANIFEST} From 4f7a84e467fc2da449d53695527866468dadb79a Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 11 Apr 2025 21:26:08 -0700 Subject: [PATCH 05/12] [cherry-pick-me]: use kw args for Test* dataclasses --- .../connector_builder_handler.py | 4 +- .../test_connector_builder_handler.py | 24 ++++- .../connector_builder/test_message_grouper.py | 87 +++++++++++++++---- .../test_yaml_declarative_source.py | 6 +- .../file_based/scenarios/check_scenarios.py | 6 +- .../file_based/scenarios/scenario_builder.py | 30 +++---- 6 files changed, 115 insertions(+), 42 deletions(-) diff --git a/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte_cdk/connector_builder/connector_builder_handler.py index 220188d34..d82fc77fb 100644 --- a/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -86,7 +86,9 @@ def read_stream( ) -> AirbyteMessage: try: test_read_handler = TestReader( - limits.max_pages_per_slice, limits.max_slices, limits.max_records + max_pages_per_slice=limits.max_pages_per_slice, + max_slices=limits.max_slices, + max_record_limit=limits.max_records, ) # The connector builder only supports a single stream stream_name = configured_catalog.streams[0].stream.name diff --git a/unit_tests/connector_builder/test_connector_builder_handler.py b/unit_tests/connector_builder/test_connector_builder_handler.py index e65c2d69f..a5caef5c4 100644 --- a/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/unit_tests/connector_builder/test_connector_builder_handler.py @@ -1069,7 +1069,11 @@ def test_read_source(mock_http_stream): max_records = 100 max_pages_per_slice = 2 max_slices = 3 - limits = TestLimits(max_records, max_pages_per_slice, max_slices) + limits = TestLimits( + max_records=max_records, + max_pages_per_slice=max_pages_per_slice, + max_slices=max_slices, + ) catalog = ConfiguredAirbyteCatalog( streams=[ @@ -1116,7 +1120,11 @@ def test_read_source_single_page_single_slice(mock_http_stream): max_records = 100 max_pages_per_slice = 1 max_slices = 1 - limits = TestLimits(max_records, max_pages_per_slice, max_slices) + limits = TestLimits( + max_records=max_records, + max_pages_per_slice=max_pages_per_slice, + max_slices=max_slices, + ) catalog = ConfiguredAirbyteCatalog( streams=[ @@ -1200,7 +1208,11 @@ def test_handle_read_external_requests(deployment_mode, url_base, expected_error endpoints when running on Cloud or OSS deployments """ - limits = TestLimits(max_records=100, max_pages_per_slice=1, max_slices=1) + limits = TestLimits( + max_records=100, + max_pages_per_slice=1, + max_slices=1, + ) catalog = ConfiguredAirbyteCatalog( streams=[ @@ -1286,7 +1298,11 @@ def test_handle_read_external_oauth_request(deployment_mode, token_url, expected endpoints when running on Cloud or OSS deployments """ - limits = TestLimits(max_records=100, max_pages_per_slice=1, max_slices=1) + limits = TestLimits( + max_records=100, + max_pages_per_slice=1, + max_slices=1, + ) catalog = ConfiguredAirbyteCatalog( streams=[ diff --git a/unit_tests/connector_builder/test_message_grouper.py b/unit_tests/connector_builder/test_message_grouper.py index c40514a27..30961a85c 100644 --- a/unit_tests/connector_builder/test_message_grouper.py +++ b/unit_tests/connector_builder/test_message_grouper.py @@ -203,7 +203,10 @@ def test_get_grouped_messages(mock_entrypoint_read: Mock) -> None: ), ) - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) actual_response: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -287,7 +290,10 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: ), ) - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) actual_response: StreamRead = connector_builder_handler.run_test_read( source=mock_source, @@ -340,7 +346,11 @@ def test_get_grouped_messages_record_limit( n_records = 2 record_limit = min(request_record_limit, max_record_limit) - api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES, max_record_limit=max_record_limit) + api = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + max_record_limit=max_record_limit, + ) # this is the call we expect to raise an exception if should_fail: with pytest.raises(ValueError): @@ -404,7 +414,11 @@ def test_get_grouped_messages_default_record_limit( ) n_records = 2 - api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES, max_record_limit=max_record_limit) + api = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + max_record_limit=max_record_limit, + ) actual_response: StreamRead = api.run_test_read( source=mock_source, config=CONFIG, @@ -443,7 +457,10 @@ def test_get_grouped_messages_limit_0(mock_entrypoint_read: Mock) -> None: ] ), ) - api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + api = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) with pytest.raises(ValueError): api.run_test_read( @@ -501,7 +518,10 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read: Mock) -> None: ), ) - message_grouper = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + message_grouper = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) actual_response: StreamRead = message_grouper.run_test_read( source=mock_source, @@ -614,7 +634,10 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> No ), ) - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, @@ -658,7 +681,10 @@ def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limi ), ) - api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + api = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = api.run_test_read( source=mock_source, @@ -686,7 +712,10 @@ def test_get_grouped_messages_given_maximum_number_of_pages_then_test_read_limit ), ) - api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + api = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = api.run_test_read( source=mock_source, @@ -705,7 +734,10 @@ def test_read_stream_returns_error_if_stream_does_not_exist() -> None: full_config: Mapping[str, Any] = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}} - message_grouper = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + message_grouper = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) actual_response = message_grouper.run_test_read( source=mock_source, config=full_config, @@ -730,7 +762,10 @@ def test_given_control_message_then_stream_read_has_config_update( + [connector_configuration_control_message(1, updated_config)] ), ) - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -760,7 +795,10 @@ def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_em ] ), ) - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -788,7 +826,10 @@ def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_ha ] ), ) - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -805,7 +846,10 @@ def test_given_auxiliary_requests_then_return_auxiliary_request(mock_entrypoint_ mock_entrypoint_read, iter(any_request_and_response_with_a_record() + [auxiliary_request_log_message()]), ) - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -819,7 +863,10 @@ def test_given_auxiliary_requests_then_return_auxiliary_request(mock_entrypoint_ @patch("airbyte_cdk.connector_builder.test_reader.reader.AirbyteEntrypoint.read") def test_given_no_slices_then_return_empty_slices(mock_entrypoint_read: Mock) -> None: mock_source = make_mock_source(mock_entrypoint_read, iter([auxiliary_request_log_message()])) - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -845,7 +892,10 @@ def test_given_pk_then_ensure_pk_is_pass_to_schema_inferrence(mock_entrypoint_re mock_source.streams.return_value = [Mock()] mock_source.streams.return_value[0].primary_key = [["id"]] mock_source.streams.return_value[0].cursor_field = _NO_CURSOR_FIELD - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, @@ -874,7 +924,10 @@ def test_given_cursor_field_then_ensure_cursor_field_is_pass_to_schema_inferrenc mock_source.streams.return_value = [Mock()] mock_source.streams.return_value[0].primary_key = _NO_PK mock_source.streams.return_value[0].cursor_field = ["date"] - connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = TestReader( + max_pages_per_slice=MAX_PAGES_PER_SLICE, + max_slices=MAX_SLICES, + ) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, diff --git a/unit_tests/sources/declarative/test_yaml_declarative_source.py b/unit_tests/sources/declarative/test_yaml_declarative_source.py index c4e5f0bfd..7f2b1aa7e 100644 --- a/unit_tests/sources/declarative/test_yaml_declarative_source.py +++ b/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -86,7 +86,7 @@ def test_source_is_created_if_toplevel_fields_are_known(self): type: CheckStream stream_names: ["lists"] """ - temporary_file = TestFileContent(content) + temporary_file = TestFileContent(content=content) MockYamlDeclarativeSource(temporary_file.filename) def test_source_fails_for_invalid_yaml(self): @@ -104,7 +104,7 @@ def test_source_fails_for_invalid_yaml(self): type: CheckStream stream_names: ["lists"] """ - temporary_file = TestFileContent(content) + temporary_file = TestFileContent(content=content) with pytest.raises(ParserError): MockYamlDeclarativeSource(temporary_file.filename) @@ -127,7 +127,7 @@ def test_source_with_missing_reference_fails(self): type: CheckStream stream_names: ["lists"] """ - temporary_file = TestFileContent(content) + temporary_file = TestFileContent(content=content) with pytest.raises(UndefinedReferenceException): MockYamlDeclarativeSource(temporary_file.filename) diff --git a/unit_tests/sources/file_based/scenarios/check_scenarios.py b/unit_tests/sources/file_based/scenarios/check_scenarios.py index 9a235b9e6..d4958545c 100644 --- a/unit_tests/sources/file_based/scenarios/check_scenarios.py +++ b/unit_tests/sources/file_based/scenarios/check_scenarios.py @@ -140,7 +140,8 @@ .set_source_builder( _base_failure_scenario.source_builder.copy().set_stream_reader( TestErrorListMatchingFilesInMemoryFilesStreamReader( - files=_base_failure_scenario.source_builder._files, file_type="csv" + files=_base_failure_scenario.source_builder._files, + file_type="csv", ) ) ) @@ -154,7 +155,8 @@ .set_source_builder( _base_failure_scenario.source_builder.copy().set_stream_reader( TestErrorOpenFileInMemoryFilesStreamReader( - files=_base_failure_scenario.source_builder._files, file_type="csv" + files=_base_failure_scenario.source_builder._files, + file_type="csv", ) ) ) diff --git a/unit_tests/sources/file_based/scenarios/scenario_builder.py b/unit_tests/sources/file_based/scenarios/scenario_builder.py index f919622e8..c106e18e1 100644 --- a/unit_tests/sources/file_based/scenarios/scenario_builder.py +++ b/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -240,21 +240,21 @@ def build(self) -> "TestScenario[SourceType]": state, ) return TestScenario( - self._name, - self._config, - source, - self._expected_spec, - self._expected_check_status, - self._expected_catalog, - self._expected_logs, - self._expected_records, - self._expected_check_error, - self._expected_discover_error, - self._expected_read_error, - self._incremental_scenario_config, - self._expected_analytics, - self._log_levels, - self._catalog, + name=self._name, + config=self._config, + source=source, + expected_spec=self._expected_spec, + expected_check_status=self._expected_check_status, + expected_catalog=self._expected_catalog, + expected_logs=self._expected_logs, + expected_records=self._expected_records, + expected_check_error=self._expected_check_error, + expected_discover_error=self._expected_discover_error, + expected_read_error=self._expected_read_error, + incremental_scenario_config=self._incremental_scenario_config, + expected_analytics=self._expected_analytics, + log_levels=self._log_levels, + catalog=self._catalog, ) def _configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]]: From 18ff2e8fc23a17dcf648e493d5d6cc9d072318cd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 12 Apr 2025 23:27:22 +0000 Subject: [PATCH 06/12] Use ClassVar[bool] for __test__ instead of kw_only=True Co-Authored-By: Aaron Steers --- airbyte_cdk/connector_builder/connector_builder_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte_cdk/connector_builder/connector_builder_handler.py index d82fc77fb..72f1fcd2a 100644 --- a/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -4,7 +4,7 @@ from dataclasses import asdict, dataclass, field -from typing import Any, Dict, List, Mapping +from typing import Any, ClassVar, Dict, List, Mapping from airbyte_cdk.connector_builder.test_reader import TestReader from airbyte_cdk.models import ( @@ -35,9 +35,9 @@ MAX_STREAMS_KEY = "max_streams" -@dataclass(kw_only=True) +@dataclass class TestLimits: - __test__: bool = False # Prevent pytest from treating this as a test case, despite its name + __test__: ClassVar[bool] = False # Prevent pytest from treating this as a test case, despite its name max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS) max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE) From 40fddcc5ebc5925c77ac15c8af2c8066487c1b82 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 12 Apr 2025 23:30:00 +0000 Subject: [PATCH 07/12] Use ClassVar[bool] for __test__ attributes instead of kw_only=True Co-Authored-By: Aaron Steers --- .../connector_builder/connector_builder_handler.py | 4 +++- airbyte_cdk/connector_builder/test_reader/reader.py | 4 ++-- .../sources/declarative/parsers/testing_components.py | 10 +++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte_cdk/connector_builder/connector_builder_handler.py index 72f1fcd2a..727be1590 100644 --- a/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -37,7 +37,9 @@ @dataclass class TestLimits: - __test__: ClassVar[bool] = False # Prevent pytest from treating this as a test case, despite its name + __test__: ClassVar[bool] = ( + False # Prevent pytest from treating this as a test case, despite its name + ) max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS) max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE) diff --git a/airbyte_cdk/connector_builder/test_reader/reader.py b/airbyte_cdk/connector_builder/test_reader/reader.py index 3955d7a3b..4498f066f 100644 --- a/airbyte_cdk/connector_builder/test_reader/reader.py +++ b/airbyte_cdk/connector_builder/test_reader/reader.py @@ -4,7 +4,7 @@ import logging -from typing import Any, Dict, Iterator, List, Mapping, Optional, Union +from typing import Any, ClassVar, Dict, Iterator, List, Mapping, Optional, Union from airbyte_cdk.connector_builder.models import ( AuxiliaryRequest, @@ -66,7 +66,7 @@ class TestReader: """ - __test__: bool = False # Prevent pytest from treating this as a test case, despite its name + __test__: ClassVar[bool] = False # Prevent pytest from treating this as a test case, despite its name logger = logging.getLogger("airbyte.connector-builder") diff --git a/unit_tests/sources/declarative/parsers/testing_components.py b/unit_tests/sources/declarative/parsers/testing_components.py index 545bf1e2f..bf32d5ca0 100644 --- a/unit_tests/sources/declarative/parsers/testing_components.py +++ b/unit_tests/sources/declarative/parsers/testing_components.py @@ -3,7 +3,7 @@ # from dataclasses import dataclass, field -from typing import List, Optional +from typing import ClassVar, List, Optional from airbyte_cdk.sources.declarative.extractors import DpathExtractor from airbyte_cdk.sources.declarative.partition_routers import SubstreamPartitionRouter @@ -15,13 +15,13 @@ ) -@dataclass(kw_only=True) +@dataclass class TestingSomeComponent(DefaultErrorHandler): """ A basic test class with various field permutations used to test manifests with custom components """ - __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name subcomponent_field_with_hint: DpathExtractor = field( default_factory=lambda: DpathExtractor(field_path=[], config={}, parameters={}) @@ -33,13 +33,13 @@ class TestingSomeComponent(DefaultErrorHandler): paginator: DefaultPaginator = None -@dataclass(kw_only=True) +@dataclass class TestingCustomSubstreamPartitionRouter(SubstreamPartitionRouter): """ A test class based on a SubstreamPartitionRouter used for testing manifests that use custom components. """ - __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name custom_field: str custom_pagination_strategy: PaginationStrategy From e2c69dfb71bc43ada5e907f61a2aeb1956cea698 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 12 Apr 2025 23:31:32 +0000 Subject: [PATCH 08/12] Update more classes to use ClassVar[bool] for __test__ attributes Co-Authored-By: Aaron Steers --- .../sources/declarative/decoders/test_composite_decoder.py | 4 ++-- .../sources/declarative/test_yaml_declarative_source.py | 3 ++- unit_tests/sources/file_based/scenarios/scenario_builder.py | 4 ++-- .../sources/file_based/test_file_based_stream_reader.py | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/unit_tests/sources/declarative/decoders/test_composite_decoder.py b/unit_tests/sources/declarative/decoders/test_composite_decoder.py index 03c947637..918010174 100644 --- a/unit_tests/sources/declarative/decoders/test_composite_decoder.py +++ b/unit_tests/sources/declarative/decoders/test_composite_decoder.py @@ -8,7 +8,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer from io import BytesIO, StringIO from threading import Thread -from typing import Iterable +from typing import ClassVar, Iterable from unittest.mock import Mock, patch import pytest @@ -259,7 +259,7 @@ def test_composite_raw_decoder_csv_parser_values(requests_mock, encoding: str, d class TestServer(BaseHTTPRequestHandler): - __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name def do_GET(self) -> None: self.send_response(200) diff --git a/unit_tests/sources/declarative/test_yaml_declarative_source.py b/unit_tests/sources/declarative/test_yaml_declarative_source.py index 7f2b1aa7e..169669620 100644 --- a/unit_tests/sources/declarative/test_yaml_declarative_source.py +++ b/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -5,6 +5,7 @@ import logging import os import tempfile +from typing import ClassVar import pytest from yaml.parser import ParserError @@ -133,7 +134,7 @@ def test_source_with_missing_reference_fails(self): class TestFileContent: - __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name def __init__(self, content): self.file = tempfile.NamedTemporaryFile(mode="w", delete=False) diff --git a/unit_tests/sources/file_based/scenarios/scenario_builder.py b/unit_tests/sources/file_based/scenarios/scenario_builder.py index c106e18e1..6d7568adf 100644 --- a/unit_tests/sources/file_based/scenarios/scenario_builder.py +++ b/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from copy import deepcopy from dataclasses import dataclass, field -from typing import Any, Generic, List, Mapping, Optional, Set, Tuple, Type, TypeVar +from typing import Any, ClassVar, Generic, List, Mapping, Optional, Set, Tuple, Type, TypeVar from airbyte_cdk.models import ( AirbyteAnalyticsTraceMessage, @@ -42,7 +42,7 @@ def build( class TestScenario(Generic[SourceType]): - __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name def __init__( self, diff --git a/unit_tests/sources/file_based/test_file_based_stream_reader.py b/unit_tests/sources/file_based/test_file_based_stream_reader.py index 366599505..347c51246 100644 --- a/unit_tests/sources/file_based/test_file_based_stream_reader.py +++ b/unit_tests/sources/file_based/test_file_based_stream_reader.py @@ -5,7 +5,7 @@ import logging from datetime import datetime from io import IOBase -from typing import Any, Dict, Iterable, List, Mapping, Optional, Set +from typing import Any, ClassVar, Dict, Iterable, List, Mapping, Optional, Set import pytest from pydantic.v1 import AnyUrl @@ -62,7 +62,7 @@ class TestStreamReader(AbstractFileBasedStreamReader): - __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name @property def config(self) -> Optional[AbstractFileBasedSpec]: @@ -102,7 +102,7 @@ def identities_schema(self) -> Dict[str, Any]: class TestSpec(AbstractFileBasedSpec): - __test__ = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name @classmethod def documentation_url(cls) -> AnyUrl: From e79e901e7fdb514c84beb5fb88b8135f309505e5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 12 Apr 2025 23:31:33 +0000 Subject: [PATCH 09/12] Apply ruff formatting Co-Authored-By: Aaron Steers --- .../declarative/decoders/test_composite_decoder.py | 4 +++- .../sources/declarative/test_yaml_declarative_source.py | 4 +++- .../sources/file_based/scenarios/scenario_builder.py | 4 +++- .../sources/file_based/test_file_based_stream_reader.py | 8 ++++++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/unit_tests/sources/declarative/decoders/test_composite_decoder.py b/unit_tests/sources/declarative/decoders/test_composite_decoder.py index 918010174..a4fde055e 100644 --- a/unit_tests/sources/declarative/decoders/test_composite_decoder.py +++ b/unit_tests/sources/declarative/decoders/test_composite_decoder.py @@ -259,7 +259,9 @@ def test_composite_raw_decoder_csv_parser_values(requests_mock, encoding: str, d class TestServer(BaseHTTPRequestHandler): - __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = ( + False # Prevent pytest from thinking that this is a test class, despite the name + ) def do_GET(self) -> None: self.send_response(200) diff --git a/unit_tests/sources/declarative/test_yaml_declarative_source.py b/unit_tests/sources/declarative/test_yaml_declarative_source.py index 169669620..89c47b5bf 100644 --- a/unit_tests/sources/declarative/test_yaml_declarative_source.py +++ b/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -134,7 +134,9 @@ def test_source_with_missing_reference_fails(self): class TestFileContent: - __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = ( + False # Prevent pytest from thinking that this is a test class, despite the name + ) def __init__(self, content): self.file = tempfile.NamedTemporaryFile(mode="w", delete=False) diff --git a/unit_tests/sources/file_based/scenarios/scenario_builder.py b/unit_tests/sources/file_based/scenarios/scenario_builder.py index 6d7568adf..7fa6282b1 100644 --- a/unit_tests/sources/file_based/scenarios/scenario_builder.py +++ b/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -42,7 +42,9 @@ def build( class TestScenario(Generic[SourceType]): - __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = ( + False # Prevent pytest from thinking that this is a test class, despite the name + ) def __init__( self, diff --git a/unit_tests/sources/file_based/test_file_based_stream_reader.py b/unit_tests/sources/file_based/test_file_based_stream_reader.py index 347c51246..ef5cd754a 100644 --- a/unit_tests/sources/file_based/test_file_based_stream_reader.py +++ b/unit_tests/sources/file_based/test_file_based_stream_reader.py @@ -62,7 +62,9 @@ class TestStreamReader(AbstractFileBasedStreamReader): - __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = ( + False # Prevent pytest from thinking that this is a test class, despite the name + ) @property def config(self) -> Optional[AbstractFileBasedSpec]: @@ -102,7 +104,9 @@ def identities_schema(self) -> Dict[str, Any]: class TestSpec(AbstractFileBasedSpec): - __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = ( + False # Prevent pytest from thinking that this is a test class, despite the name + ) @classmethod def documentation_url(cls) -> AnyUrl: From c26345e1be108b891883feeac98209f15134fb2b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 12 Apr 2025 23:31:45 +0000 Subject: [PATCH 10/12] Fix remaining files with ruff formatting Co-Authored-By: Aaron Steers --- airbyte_cdk/connector_builder/test_reader/reader.py | 4 +++- .../sources/declarative/parsers/testing_components.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/connector_builder/test_reader/reader.py b/airbyte_cdk/connector_builder/test_reader/reader.py index 4498f066f..8b38d7a09 100644 --- a/airbyte_cdk/connector_builder/test_reader/reader.py +++ b/airbyte_cdk/connector_builder/test_reader/reader.py @@ -66,7 +66,9 @@ class TestReader: """ - __test__: ClassVar[bool] = False # Prevent pytest from treating this as a test case, despite its name + __test__: ClassVar[bool] = ( + False # Prevent pytest from treating this as a test case, despite its name + ) logger = logging.getLogger("airbyte.connector-builder") diff --git a/unit_tests/sources/declarative/parsers/testing_components.py b/unit_tests/sources/declarative/parsers/testing_components.py index bf32d5ca0..582184020 100644 --- a/unit_tests/sources/declarative/parsers/testing_components.py +++ b/unit_tests/sources/declarative/parsers/testing_components.py @@ -21,7 +21,9 @@ class TestingSomeComponent(DefaultErrorHandler): A basic test class with various field permutations used to test manifests with custom components """ - __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = ( + False # Prevent pytest from thinking that this is a test class, despite the name + ) subcomponent_field_with_hint: DpathExtractor = field( default_factory=lambda: DpathExtractor(field_path=[], config={}, parameters={}) @@ -39,7 +41,9 @@ class TestingCustomSubstreamPartitionRouter(SubstreamPartitionRouter): A test class based on a SubstreamPartitionRouter used for testing manifests that use custom components. """ - __test__: ClassVar[bool] = False # Prevent pytest from thinking that this is a test class, despite the name + __test__: ClassVar[bool] = ( + False # Prevent pytest from thinking that this is a test class, despite the name + ) custom_field: str custom_pagination_strategy: PaginationStrategy From eb0f643e147b4fe07249beef050a03cac2eb3616 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Sat, 12 Apr 2025 16:40:30 -0700 Subject: [PATCH 11/12] addl cleanup --- .../connector_builder_handler.py | 4 +- .../connector_builder/test_reader/reader.py | 4 +- pytest.ini | 1 + .../test_connector_builder_handler.py | 30 ++----- .../connector_builder/test_message_grouper.py | 87 ++++--------------- .../decoders/test_composite_decoder.py | 4 +- .../declarative/parsers/testing_components.py | 8 +- .../test_yaml_declarative_source.py | 10 +-- .../file_based/scenarios/check_scenarios.py | 6 +- .../file_based/scenarios/scenario_builder.py | 4 +- .../test_file_based_stream_reader.py | 4 +- 11 files changed, 36 insertions(+), 126 deletions(-) diff --git a/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte_cdk/connector_builder/connector_builder_handler.py index 727be1590..6daceb15a 100644 --- a/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -37,9 +37,7 @@ @dataclass class TestLimits: - __test__: ClassVar[bool] = ( - False # Prevent pytest from treating this as a test case, despite its name - ) + __test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS) max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE) diff --git a/airbyte_cdk/connector_builder/test_reader/reader.py b/airbyte_cdk/connector_builder/test_reader/reader.py index 8b38d7a09..a125047b4 100644 --- a/airbyte_cdk/connector_builder/test_reader/reader.py +++ b/airbyte_cdk/connector_builder/test_reader/reader.py @@ -66,9 +66,7 @@ class TestReader: """ - __test__: ClassVar[bool] = ( - False # Prevent pytest from treating this as a test case, despite its name - ) + __test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name logger = logging.getLogger("airbyte.connector-builder") diff --git a/pytest.ini b/pytest.ini index 2aaee685b..d6b4b4683 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,3 +8,4 @@ filterwarnings = markers = slow: mark tests as slow asyncio: mark test as asyncio + requires_creds: mark test as requiring credentials diff --git a/unit_tests/connector_builder/test_connector_builder_handler.py b/unit_tests/connector_builder/test_connector_builder_handler.py index a5caef5c4..2e786af8c 100644 --- a/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/unit_tests/connector_builder/test_connector_builder_handler.py @@ -983,11 +983,7 @@ def test_create_source() -> None: max_records = 3 max_pages_per_slice = 2 max_slices = 1 - limits = TestLimits( - max_records=max_records, - max_pages_per_slice=max_pages_per_slice, - max_slices=max_slices, - ) + limits = TestLimits(max_records, max_pages_per_slice, max_slices) config = {"__injected_declarative_manifest": MANIFEST} @@ -1069,11 +1065,7 @@ def test_read_source(mock_http_stream): max_records = 100 max_pages_per_slice = 2 max_slices = 3 - limits = TestLimits( - max_records=max_records, - max_pages_per_slice=max_pages_per_slice, - max_slices=max_slices, - ) + limits = TestLimits(max_records, max_pages_per_slice, max_slices) catalog = ConfiguredAirbyteCatalog( streams=[ @@ -1120,11 +1112,7 @@ def test_read_source_single_page_single_slice(mock_http_stream): max_records = 100 max_pages_per_slice = 1 max_slices = 1 - limits = TestLimits( - max_records=max_records, - max_pages_per_slice=max_pages_per_slice, - max_slices=max_slices, - ) + limits = TestLimits(max_records, max_pages_per_slice, max_slices) catalog = ConfiguredAirbyteCatalog( streams=[ @@ -1208,11 +1196,7 @@ def test_handle_read_external_requests(deployment_mode, url_base, expected_error endpoints when running on Cloud or OSS deployments """ - limits = TestLimits( - max_records=100, - max_pages_per_slice=1, - max_slices=1, - ) + limits = TestLimits(max_records=100, max_pages_per_slice=1, max_slices=1) catalog = ConfiguredAirbyteCatalog( streams=[ @@ -1298,11 +1282,7 @@ def test_handle_read_external_oauth_request(deployment_mode, token_url, expected endpoints when running on Cloud or OSS deployments """ - limits = TestLimits( - max_records=100, - max_pages_per_slice=1, - max_slices=1, - ) + limits = TestLimits(max_records=100, max_pages_per_slice=1, max_slices=1) catalog = ConfiguredAirbyteCatalog( streams=[ diff --git a/unit_tests/connector_builder/test_message_grouper.py b/unit_tests/connector_builder/test_message_grouper.py index 30961a85c..c40514a27 100644 --- a/unit_tests/connector_builder/test_message_grouper.py +++ b/unit_tests/connector_builder/test_message_grouper.py @@ -203,10 +203,7 @@ def test_get_grouped_messages(mock_entrypoint_read: Mock) -> None: ), ) - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -290,10 +287,7 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: ), ) - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = connector_builder_handler.run_test_read( source=mock_source, @@ -346,11 +340,7 @@ def test_get_grouped_messages_record_limit( n_records = 2 record_limit = min(request_record_limit, max_record_limit) - api = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - max_record_limit=max_record_limit, - ) + api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES, max_record_limit=max_record_limit) # this is the call we expect to raise an exception if should_fail: with pytest.raises(ValueError): @@ -414,11 +404,7 @@ def test_get_grouped_messages_default_record_limit( ) n_records = 2 - api = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - max_record_limit=max_record_limit, - ) + api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES, max_record_limit=max_record_limit) actual_response: StreamRead = api.run_test_read( source=mock_source, config=CONFIG, @@ -457,10 +443,7 @@ def test_get_grouped_messages_limit_0(mock_entrypoint_read: Mock) -> None: ] ), ) - api = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) with pytest.raises(ValueError): api.run_test_read( @@ -518,10 +501,7 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read: Mock) -> None: ), ) - message_grouper = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + message_grouper = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = message_grouper.run_test_read( source=mock_source, @@ -634,10 +614,7 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> No ), ) - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, @@ -681,10 +658,7 @@ def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limi ), ) - api = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = api.run_test_read( source=mock_source, @@ -712,10 +686,7 @@ def test_get_grouped_messages_given_maximum_number_of_pages_then_test_read_limit ), ) - api = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + api = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = api.run_test_read( source=mock_source, @@ -734,10 +705,7 @@ def test_read_stream_returns_error_if_stream_does_not_exist() -> None: full_config: Mapping[str, Any] = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}} - message_grouper = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + message_grouper = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response = message_grouper.run_test_read( source=mock_source, config=full_config, @@ -762,10 +730,7 @@ def test_given_control_message_then_stream_read_has_config_update( + [connector_configuration_control_message(1, updated_config)] ), ) - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -795,10 +760,7 @@ def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_em ] ), ) - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -826,10 +788,7 @@ def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_ha ] ), ) - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -846,10 +805,7 @@ def test_given_auxiliary_requests_then_return_auxiliary_request(mock_entrypoint_ mock_entrypoint_read, iter(any_request_and_response_with_a_record() + [auxiliary_request_log_message()]), ) - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -863,10 +819,7 @@ def test_given_auxiliary_requests_then_return_auxiliary_request(mock_entrypoint_ @patch("airbyte_cdk.connector_builder.test_reader.reader.AirbyteEntrypoint.read") def test_given_no_slices_then_return_empty_slices(mock_entrypoint_read: Mock) -> None: mock_source = make_mock_source(mock_entrypoint_read, iter([auxiliary_request_log_message()])) - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, config=CONFIG, @@ -892,10 +845,7 @@ def test_given_pk_then_ensure_pk_is_pass_to_schema_inferrence(mock_entrypoint_re mock_source.streams.return_value = [Mock()] mock_source.streams.return_value[0].primary_key = [["id"]] mock_source.streams.return_value[0].cursor_field = _NO_CURSOR_FIELD - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, @@ -924,10 +874,7 @@ def test_given_cursor_field_then_ensure_cursor_field_is_pass_to_schema_inferrenc mock_source.streams.return_value = [Mock()] mock_source.streams.return_value[0].primary_key = _NO_PK mock_source.streams.return_value[0].cursor_field = ["date"] - connector_builder_handler = TestReader( - max_pages_per_slice=MAX_PAGES_PER_SLICE, - max_slices=MAX_SLICES, - ) + connector_builder_handler = TestReader(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.run_test_read( source=mock_source, diff --git a/unit_tests/sources/declarative/decoders/test_composite_decoder.py b/unit_tests/sources/declarative/decoders/test_composite_decoder.py index a4fde055e..cd0a562ae 100644 --- a/unit_tests/sources/declarative/decoders/test_composite_decoder.py +++ b/unit_tests/sources/declarative/decoders/test_composite_decoder.py @@ -259,9 +259,7 @@ def test_composite_raw_decoder_csv_parser_values(requests_mock, encoding: str, d class TestServer(BaseHTTPRequestHandler): - __test__: ClassVar[bool] = ( - False # Prevent pytest from thinking that this is a test class, despite the name - ) + __test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name def do_GET(self) -> None: self.send_response(200) diff --git a/unit_tests/sources/declarative/parsers/testing_components.py b/unit_tests/sources/declarative/parsers/testing_components.py index 582184020..ab9ae5346 100644 --- a/unit_tests/sources/declarative/parsers/testing_components.py +++ b/unit_tests/sources/declarative/parsers/testing_components.py @@ -21,9 +21,7 @@ class TestingSomeComponent(DefaultErrorHandler): A basic test class with various field permutations used to test manifests with custom components """ - __test__: ClassVar[bool] = ( - False # Prevent pytest from thinking that this is a test class, despite the name - ) + __test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name subcomponent_field_with_hint: DpathExtractor = field( default_factory=lambda: DpathExtractor(field_path=[], config={}, parameters={}) @@ -41,9 +39,7 @@ class TestingCustomSubstreamPartitionRouter(SubstreamPartitionRouter): A test class based on a SubstreamPartitionRouter used for testing manifests that use custom components. """ - __test__: ClassVar[bool] = ( - False # Prevent pytest from thinking that this is a test class, despite the name - ) + __test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name custom_field: str custom_pagination_strategy: PaginationStrategy diff --git a/unit_tests/sources/declarative/test_yaml_declarative_source.py b/unit_tests/sources/declarative/test_yaml_declarative_source.py index 89c47b5bf..846547eec 100644 --- a/unit_tests/sources/declarative/test_yaml_declarative_source.py +++ b/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -87,7 +87,7 @@ def test_source_is_created_if_toplevel_fields_are_known(self): type: CheckStream stream_names: ["lists"] """ - temporary_file = TestFileContent(content=content) + temporary_file = TestFileContent(content) MockYamlDeclarativeSource(temporary_file.filename) def test_source_fails_for_invalid_yaml(self): @@ -105,7 +105,7 @@ def test_source_fails_for_invalid_yaml(self): type: CheckStream stream_names: ["lists"] """ - temporary_file = TestFileContent(content=content) + temporary_file = TestFileContent(content) with pytest.raises(ParserError): MockYamlDeclarativeSource(temporary_file.filename) @@ -128,15 +128,13 @@ def test_source_with_missing_reference_fails(self): type: CheckStream stream_names: ["lists"] """ - temporary_file = TestFileContent(content=content) + temporary_file = TestFileContent(content) with pytest.raises(UndefinedReferenceException): MockYamlDeclarativeSource(temporary_file.filename) class TestFileContent: - __test__: ClassVar[bool] = ( - False # Prevent pytest from thinking that this is a test class, despite the name - ) + __test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name def __init__(self, content): self.file = tempfile.NamedTemporaryFile(mode="w", delete=False) diff --git a/unit_tests/sources/file_based/scenarios/check_scenarios.py b/unit_tests/sources/file_based/scenarios/check_scenarios.py index d4958545c..9a235b9e6 100644 --- a/unit_tests/sources/file_based/scenarios/check_scenarios.py +++ b/unit_tests/sources/file_based/scenarios/check_scenarios.py @@ -140,8 +140,7 @@ .set_source_builder( _base_failure_scenario.source_builder.copy().set_stream_reader( TestErrorListMatchingFilesInMemoryFilesStreamReader( - files=_base_failure_scenario.source_builder._files, - file_type="csv", + files=_base_failure_scenario.source_builder._files, file_type="csv" ) ) ) @@ -155,8 +154,7 @@ .set_source_builder( _base_failure_scenario.source_builder.copy().set_stream_reader( TestErrorOpenFileInMemoryFilesStreamReader( - files=_base_failure_scenario.source_builder._files, - file_type="csv", + files=_base_failure_scenario.source_builder._files, file_type="csv" ) ) ) diff --git a/unit_tests/sources/file_based/scenarios/scenario_builder.py b/unit_tests/sources/file_based/scenarios/scenario_builder.py index 7fa6282b1..93e8f952a 100644 --- a/unit_tests/sources/file_based/scenarios/scenario_builder.py +++ b/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -42,9 +42,7 @@ def build( class TestScenario(Generic[SourceType]): - __test__: ClassVar[bool] = ( - False # Prevent pytest from thinking that this is a test class, despite the name - ) + __test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name def __init__( self, diff --git a/unit_tests/sources/file_based/test_file_based_stream_reader.py b/unit_tests/sources/file_based/test_file_based_stream_reader.py index ef5cd754a..ace2999e0 100644 --- a/unit_tests/sources/file_based/test_file_based_stream_reader.py +++ b/unit_tests/sources/file_based/test_file_based_stream_reader.py @@ -62,9 +62,7 @@ class TestStreamReader(AbstractFileBasedStreamReader): - __test__: ClassVar[bool] = ( - False # Prevent pytest from thinking that this is a test class, despite the name - ) + __test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name @property def config(self) -> Optional[AbstractFileBasedSpec]: From 4f84c14dfe308de22b4c3badfde7c90f364bde12 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 14 Apr 2025 10:40:06 -0700 Subject: [PATCH 12/12] revert unnecessary changes --- .../connector_builder/connector_builder_handler.py | 11 ++--------- .../test_connector_builder_handler.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte_cdk/connector_builder/connector_builder_handler.py index 6daceb15a..6b6b31111 100644 --- a/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -53,12 +53,7 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits: max_slices = command_config.get(MAX_SLICES_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_SLICES max_records = command_config.get(MAX_RECORDS_KEY) or DEFAULT_MAXIMUM_RECORDS max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS - return TestLimits( - max_records=max_records, - max_pages_per_slice=max_pages_per_slice, - max_slices=max_slices, - max_streams=max_streams, - ) + return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams) def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource: @@ -86,9 +81,7 @@ def read_stream( ) -> AirbyteMessage: try: test_read_handler = TestReader( - max_pages_per_slice=limits.max_pages_per_slice, - max_slices=limits.max_slices, - max_record_limit=limits.max_records, + limits.max_pages_per_slice, limits.max_slices, limits.max_records ) # The connector builder only supports a single stream stream_name = configured_catalog.streams[0].stream.name diff --git a/unit_tests/connector_builder/test_connector_builder_handler.py b/unit_tests/connector_builder/test_connector_builder_handler.py index 6e5368788..d98a49a8c 100644 --- a/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/unit_tests/connector_builder/test_connector_builder_handler.py @@ -979,7 +979,7 @@ def test_get_limits( assert limits.max_slices == expected_max_slices -def test_create_source() -> None: +def test_create_source(): max_records = 3 max_pages_per_slice = 2 max_slices = 1