Skip to content
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions airbyte_cdk/connector_builder/connector_builder_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@


from dataclasses import asdict, dataclass, field
from typing import Any, Dict, List, Mapping
from typing import Any, ClassVar, Dict, List, Mapping

from airbyte_cdk.connector_builder.test_reader import TestReader
from airbyte_cdk.models import (
Expand Down Expand Up @@ -37,6 +37,8 @@

@dataclass
class TestLimits:
__test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name

max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)
Expand All @@ -51,7 +53,12 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits:
max_slices = command_config.get(MAX_SLICES_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_SLICES
max_records = command_config.get(MAX_RECORDS_KEY) or DEFAULT_MAXIMUM_RECORDS
max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS
return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
return TestLimits(
max_records=max_records,
max_pages_per_slice=max_pages_per_slice,
max_slices=max_slices,
max_streams=max_streams,
)


def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource:
Expand Down Expand Up @@ -79,7 +86,9 @@ def read_stream(
) -> AirbyteMessage:
try:
test_read_handler = TestReader(
limits.max_pages_per_slice, limits.max_slices, limits.max_records
max_pages_per_slice=limits.max_pages_per_slice,
max_slices=limits.max_slices,
max_record_limit=limits.max_records,
)
# The connector builder only supports a single stream
stream_name = configured_catalog.streams[0].stream.name
Expand Down
4 changes: 3 additions & 1 deletion airbyte_cdk/connector_builder/test_reader/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@


import logging
from typing import Any, Dict, Iterator, List, Mapping, Optional, Union
from typing import Any, ClassVar, Dict, Iterator, List, Mapping, Optional, Union

from airbyte_cdk.connector_builder.models import (
AuxiliaryRequest,
Expand Down Expand Up @@ -66,6 +66,8 @@ class TestReader:

"""

__test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name

logger = logging.getLogger("airbyte.connector-builder")

def __init__(
Expand Down
4 changes: 4 additions & 0 deletions pytest.ini
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,7 @@ log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno
log_cli_date_format=%Y-%m-%d %H:%M:%S
filterwarnings =
ignore::airbyte_cdk.sources.source.ExperimentalClassWarning
markers =
slow: mark tests as slow
asyncio: mark test as asyncio
requires_creds: mark test as requiring credentials
Original file line number Diff line number Diff line change
Expand Up @@ -979,7 +979,7 @@ def test_get_limits(
assert limits.max_slices == expected_max_slices


def test_create_source():
def test_create_source() -> None:
max_records = 3
max_pages_per_slice = 2
max_slices = 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from http.server import BaseHTTPRequestHandler, HTTPServer
from io import BytesIO, StringIO
from threading import Thread
from typing import Iterable
from typing import ClassVar, Iterable
from unittest.mock import Mock, patch

import pytest
Expand Down Expand Up @@ -259,6 +259,8 @@ def test_composite_raw_decoder_csv_parser_values(requests_mock, encoding: str, d


class TestServer(BaseHTTPRequestHandler):
__test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name

def do_GET(self) -> None:
self.send_response(200)
self.end_headers()
Expand Down
6 changes: 5 additions & 1 deletion unit_tests/sources/declarative/parsers/testing_components.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#

from dataclasses import dataclass, field
from typing import List, Optional
from typing import ClassVar, List, Optional

from airbyte_cdk.sources.declarative.extractors import DpathExtractor
from airbyte_cdk.sources.declarative.partition_routers import SubstreamPartitionRouter
Expand All @@ -21,6 +21,8 @@ class TestingSomeComponent(DefaultErrorHandler):
A basic test class with various field permutations used to test manifests with custom components
"""

__test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name

subcomponent_field_with_hint: DpathExtractor = field(
default_factory=lambda: DpathExtractor(field_path=[], config={}, parameters={})
)
Expand All @@ -37,5 +39,7 @@ class TestingCustomSubstreamPartitionRouter(SubstreamPartitionRouter):
A test class based on a SubstreamPartitionRouter used for testing manifests that use custom components.
"""

__test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name

custom_field: str
custom_pagination_strategy: PaginationStrategy
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import logging
import os
import tempfile
from typing import ClassVar

import pytest
from yaml.parser import ParserError
Expand Down Expand Up @@ -133,6 +134,8 @@ def test_source_with_missing_reference_fails(self):


class TestFileContent:
__test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name

def __init__(self, content):
self.file = tempfile.NamedTemporaryFile(mode="w", delete=False)

Expand Down
34 changes: 18 additions & 16 deletions unit_tests/sources/file_based/scenarios/scenario_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from abc import ABC, abstractmethod
from copy import deepcopy
from dataclasses import dataclass, field
from typing import Any, Generic, List, Mapping, Optional, Set, Tuple, Type, TypeVar
from typing import Any, ClassVar, Generic, List, Mapping, Optional, Set, Tuple, Type, TypeVar

from airbyte_cdk.models import (
AirbyteAnalyticsTraceMessage,
Expand Down Expand Up @@ -42,6 +42,8 @@ def build(


class TestScenario(Generic[SourceType]):
__test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name

def __init__(
self,
name: str,
Expand Down Expand Up @@ -238,21 +240,21 @@ def build(self) -> "TestScenario[SourceType]":
state,
)
return TestScenario(
self._name,
self._config,
source,
self._expected_spec,
self._expected_check_status,
self._expected_catalog,
self._expected_logs,
self._expected_records,
self._expected_check_error,
self._expected_discover_error,
self._expected_read_error,
self._incremental_scenario_config,
self._expected_analytics,
self._log_levels,
self._catalog,
name=self._name,
config=self._config,
source=source,
expected_spec=self._expected_spec,
expected_check_status=self._expected_check_status,
expected_catalog=self._expected_catalog,
expected_logs=self._expected_logs,
expected_records=self._expected_records,
expected_check_error=self._expected_check_error,
expected_discover_error=self._expected_discover_error,
expected_read_error=self._expected_read_error,
incremental_scenario_config=self._incremental_scenario_config,
expected_analytics=self._expected_analytics,
log_levels=self._log_levels,
catalog=self._catalog,
)

def _configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import logging
from datetime import datetime
from io import IOBase
from typing import Any, Dict, Iterable, List, Mapping, Optional, Set
from typing import Any, ClassVar, Dict, Iterable, List, Mapping, Optional, Set

import pytest
from pydantic.v1 import AnyUrl
Expand Down Expand Up @@ -62,6 +62,8 @@


class TestStreamReader(AbstractFileBasedStreamReader):
__test__: ClassVar[bool] = False # Tell Pytest this is not a Pytest class, despite its name

@property
def config(self) -> Optional[AbstractFileBasedSpec]:
return self._config
Expand Down Expand Up @@ -100,6 +102,10 @@ def identities_schema(self) -> Dict[str, Any]:


class TestSpec(AbstractFileBasedSpec):
__test__: ClassVar[bool] = (
False # Prevent pytest from thinking that this is a test class, despite the name
)

@classmethod
def documentation_url(cls) -> AnyUrl:
return AnyUrl(scheme="https", url="https://docs.airbyte.com/integrations/sources/test") # type: ignore
Expand Down