From eb212b3882e5148cd6ec214086cb6961ed41f05b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 19:57:42 +0000 Subject: [PATCH 01/18] Initial commit for DNS error handling changes Co-Authored-By: Aaron Steers From 283802d44c8395d1f6fbf33a24e604d2da1ebad6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 19:58:07 +0000 Subject: [PATCH 02/18] Update InvalidURL error handling to retry on temporary DNS failures Co-Authored-By: Aaron Steers --- .../streams/http/error_handlers/default_error_mapping.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py b/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py index 74840e2d2..6222b1338 100644 --- a/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +++ b/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py @@ -19,9 +19,9 @@ error_message="Invalid Protocol Schema: The endpoint that data is being requested from is using an invalid or insecure. Exception: requests.exceptions.InvalidSchema", ), InvalidURL: ErrorResolution( - response_action=ResponseAction.FAIL, - failure_type=FailureType.config_error, - error_message="Invalid URL specified: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL", + response_action=ResponseAction.RETRY, + failure_type=FailureType.transient_error, + error_message="Temporary DNS resolution error occurred. Retrying...", ), RequestException: ErrorResolution( response_action=ResponseAction.RETRY, From b3c07347a0fa765370ce5369ccddbd6e1d5abd65 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 19:59:34 +0000 Subject: [PATCH 03/18] Add test for DNS resolution error handling Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 40fdb3201..fd14c5942 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -331,6 +331,16 @@ def test_raise_on_http_errors(mocker, error): assert send_mock.call_count == stream.max_retries + 1 +def test_dns_resolution_error_retry(): + """Test that DNS resolution errors are retried""" + stream = StubBasicReadHttpStream() + error_handler = stream.get_error_handler() + resolution = error_handler.interpret_response(requests.exceptions.InvalidURL()) + + assert resolution.response_action == ResponseAction.RETRY + assert resolution.failure_type == FailureType.transient_error + + class PostHttpStream(StubBasicReadHttpStream): http_method = "POST" From 984f4eb92b090107648a677135c38909a3b30431 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 19:59:51 +0000 Subject: [PATCH 04/18] Fix import for FailureType in test Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index fd14c5942..879767535 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -20,7 +20,7 @@ from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, HttpStatusErrorHandler -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction, FailureType from airbyte_cdk.sources.streams.http.exceptions import ( DefaultBackoffException, RequestBodyException, From aef78e8e6086e30cb784f86d32cf385a89dd7094 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:00:24 +0000 Subject: [PATCH 05/18] Update DNS error test to match expected pattern Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 879767535..a5a6ec914 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -334,11 +334,11 @@ def test_raise_on_http_errors(mocker, error): def test_dns_resolution_error_retry(): """Test that DNS resolution errors are retried""" stream = StubBasicReadHttpStream() - error_handler = stream.get_error_handler() - resolution = error_handler.interpret_response(requests.exceptions.InvalidURL()) - - assert resolution.response_action == ResponseAction.RETRY - assert resolution.failure_type == FailureType.transient_error + with pytest.raises(InvalidURL) as exc: + # Test that InvalidURL is treated as transient error + stream._send_request(requests.PreparedRequest(), {}) + assert exc.value.response_action == ResponseAction.RETRY + assert exc.value.failure_type == FailureType.transient_error class PostHttpStream(StubBasicReadHttpStream): From 69620accb08f7066c4af3c35ea1ed16b23618cfe Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:00:55 +0000 Subject: [PATCH 06/18] Add InvalidURL import for DNS error test Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index a5a6ec914..b51dee1d3 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -10,6 +10,7 @@ import pytest import requests +from requests.exceptions import InvalidURL from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type from airbyte_cdk.sources.streams import CheckpointMixin From 42579cad8563f697b4c1a99abce5d05f4e00e62f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:02:01 +0000 Subject: [PATCH 07/18] Fix DNS error test to use correct method Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index b51dee1d3..b25a84fbb 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -337,7 +337,7 @@ def test_dns_resolution_error_retry(): stream = StubBasicReadHttpStream() with pytest.raises(InvalidURL) as exc: # Test that InvalidURL is treated as transient error - stream._send_request(requests.PreparedRequest(), {}) + stream._http_client.send_request("GET", "invalid_url", {}) assert exc.value.response_action == ResponseAction.RETRY assert exc.value.failure_type == FailureType.transient_error From 7a0b67b4a4bb4f4648a922389a7fd86d9477289b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:02:24 +0000 Subject: [PATCH 08/18] Fix DNS error test to use malformed URL Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index b25a84fbb..f0bccd7bf 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -337,7 +337,7 @@ def test_dns_resolution_error_retry(): stream = StubBasicReadHttpStream() with pytest.raises(InvalidURL) as exc: # Test that InvalidURL is treated as transient error - stream._http_client.send_request("GET", "invalid_url", {}) + stream._http_client.send_request("GET", "http://[invalid-url", {}) assert exc.value.response_action == ResponseAction.RETRY assert exc.value.failure_type == FailureType.transient_error From 9bac7fc6525e930d281b448c9c01970375b3edb8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:02:48 +0000 Subject: [PATCH 09/18] Fix DNS error test to use error handler Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index f0bccd7bf..315d7a417 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -335,11 +335,10 @@ def test_raise_on_http_errors(mocker, error): def test_dns_resolution_error_retry(): """Test that DNS resolution errors are retried""" stream = StubBasicReadHttpStream() - with pytest.raises(InvalidURL) as exc: - # Test that InvalidURL is treated as transient error - stream._http_client.send_request("GET", "http://[invalid-url", {}) - assert exc.value.response_action == ResponseAction.RETRY - assert exc.value.failure_type == FailureType.transient_error + error_handler = stream.get_error_handler() + resolution = error_handler.interpret_response(InvalidURL()) + assert resolution.response_action == ResponseAction.RETRY + assert resolution.failure_type == FailureType.transient_error class PostHttpStream(StubBasicReadHttpStream): From 492dc088607d027717ab8c2b9fe6001238a1a114 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:03:29 +0000 Subject: [PATCH 10/18] Add error handler to StubBasicReadHttpStream Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 315d7a417..88b41a469 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -41,6 +41,9 @@ def __init__(self, deduplicate_query_params: bool = False, **kwargs): self.resp_counter = 1 self._deduplicate_query_params = deduplicate_query_params + def get_error_handler(self) -> Optional[ErrorHandler]: + return HttpStatusErrorHandler(logging.getLogger()) + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None From 5ade37cfce68a56aeceb433ada89d73c3417b58a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:04:08 +0000 Subject: [PATCH 11/18] Create dedicated test class for DNS error handling Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 88b41a469..773d26763 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -41,9 +41,6 @@ def __init__(self, deduplicate_query_params: bool = False, **kwargs): self.resp_counter = 1 self._deduplicate_query_params = deduplicate_query_params - def get_error_handler(self) -> Optional[ErrorHandler]: - return HttpStatusErrorHandler(logging.getLogger()) - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None @@ -335,9 +332,13 @@ def test_raise_on_http_errors(mocker, error): assert send_mock.call_count == stream.max_retries + 1 +class StubHttpStreamWithErrorHandler(StubBasicReadHttpStream): + def get_error_handler(self) -> Optional[ErrorHandler]: + return HttpStatusErrorHandler(logging.getLogger()) + def test_dns_resolution_error_retry(): """Test that DNS resolution errors are retried""" - stream = StubBasicReadHttpStream() + stream = StubHttpStreamWithErrorHandler() error_handler = stream.get_error_handler() resolution = error_handler.interpret_response(InvalidURL()) assert resolution.response_action == ResponseAction.RETRY From e8541be495ca0430c82cfac339a77b261f545aac Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:04:52 +0000 Subject: [PATCH 12/18] Fix import sorting Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 773d26763..8da447962 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -10,8 +10,6 @@ import pytest import requests -from requests.exceptions import InvalidURL - from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type from airbyte_cdk.sources.streams import CheckpointMixin from airbyte_cdk.sources.streams.checkpoint import ResumableFullRefreshCursor @@ -21,7 +19,7 @@ from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, HttpStatusErrorHandler -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction, FailureType +from airbyte_cdk.sources.streams.http.error_handlers.response_models import FailureType, ResponseAction from airbyte_cdk.sources.streams.http.exceptions import ( DefaultBackoffException, RequestBodyException, @@ -30,6 +28,7 @@ from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.utils.airbyte_secrets_utils import update_secrets +from requests.exceptions import InvalidURL class StubBasicReadHttpStream(HttpStream): From 24d48c16053c17528e3ff5a694e867603611a322 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:06:32 +0000 Subject: [PATCH 13/18] Fix formatting and lint issues Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 8da447962..9f6209866 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -10,6 +10,8 @@ import pytest import requests +from requests.exceptions import InvalidURL + from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type from airbyte_cdk.sources.streams import CheckpointMixin from airbyte_cdk.sources.streams.checkpoint import ResumableFullRefreshCursor @@ -19,7 +21,10 @@ from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, HttpStatusErrorHandler -from airbyte_cdk.sources.streams.http.error_handlers.response_models import FailureType, ResponseAction +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ( + FailureType, + ResponseAction, +) from airbyte_cdk.sources.streams.http.exceptions import ( DefaultBackoffException, RequestBodyException, @@ -28,7 +33,6 @@ from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.utils.airbyte_secrets_utils import update_secrets -from requests.exceptions import InvalidURL class StubBasicReadHttpStream(HttpStream): @@ -335,6 +339,7 @@ class StubHttpStreamWithErrorHandler(StubBasicReadHttpStream): def get_error_handler(self) -> Optional[ErrorHandler]: return HttpStatusErrorHandler(logging.getLogger()) + def test_dns_resolution_error_retry(): """Test that DNS resolution errors are retried""" stream = StubHttpStreamWithErrorHandler() From 888d5a3be3fe33912f0d14030adb7726b672ac3d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:14:09 +0000 Subject: [PATCH 14/18] Add DNSResolutionError class and update error handling Co-Authored-By: Aaron Steers --- .../error_handlers/default_error_mapping.py | 12 +++++++++--- airbyte_cdk/sources/streams/http/exceptions.py | 17 +++++++++++++++++ unit_tests/sources/streams/http/test_http.py | 13 ++++++++++++- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py b/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py index 6222b1338..78dbb3ae5 100644 --- a/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +++ b/airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py @@ -11,17 +11,23 @@ ErrorResolution, ResponseAction, ) +from airbyte_cdk.sources.streams.http.exceptions import DNSResolutionError DEFAULT_ERROR_MAPPING: Mapping[Union[int, str, Type[Exception]], ErrorResolution] = { + DNSResolutionError: ErrorResolution( + response_action=ResponseAction.RETRY, + failure_type=FailureType.transient_error, + error_message="Temporary DNS resolution error occurred. Retrying...", + ), InvalidSchema: ErrorResolution( response_action=ResponseAction.FAIL, failure_type=FailureType.config_error, error_message="Invalid Protocol Schema: The endpoint that data is being requested from is using an invalid or insecure. Exception: requests.exceptions.InvalidSchema", ), InvalidURL: ErrorResolution( - response_action=ResponseAction.RETRY, - failure_type=FailureType.transient_error, - error_message="Temporary DNS resolution error occurred. Retrying...", + response_action=ResponseAction.FAIL, + failure_type=FailureType.config_error, + error_message="Invalid URL specified: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL", ), RequestException: ErrorResolution( response_action=ResponseAction.RETRY, diff --git a/airbyte_cdk/sources/streams/http/exceptions.py b/airbyte_cdk/sources/streams/http/exceptions.py index ee4687626..40d3300ec 100644 --- a/airbyte_cdk/sources/streams/http/exceptions.py +++ b/airbyte_cdk/sources/streams/http/exceptions.py @@ -59,3 +59,20 @@ class DefaultBackoffException(BaseBackoffException): class RateLimitBackoffException(BaseBackoffException): pass + + +class DNSResolutionError(BaseBackoffException): + """ + Raised when a DNS resolution error occurs. + This is different from InvalidURL which is raised for malformed URLs. + """ + + def __init__( + self, + url: str, + request: requests.PreparedRequest, + response: Optional[Union[requests.Response, Exception]], + error_message: str = "", + ): + self.url = url + super().__init__(request=request, response=response, error_message=error_message or f"Failed to resolve DNS for URL: {url}") diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 9f6209866..5deaa9866 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -344,10 +344,21 @@ def test_dns_resolution_error_retry(): """Test that DNS resolution errors are retried""" stream = StubHttpStreamWithErrorHandler() error_handler = stream.get_error_handler() - resolution = error_handler.interpret_response(InvalidURL()) + request = requests.PreparedRequest() + request.url = "https://example.com" + dns_error = DNSResolutionError(url="https://example.com", request=request, response=Exception("DNS lookup failed")) + resolution = error_handler.interpret_response(dns_error) assert resolution.response_action == ResponseAction.RETRY assert resolution.failure_type == FailureType.transient_error +def test_invalid_url_fails(): + """Test that invalid URLs fail immediately""" + stream = StubHttpStreamWithErrorHandler() + error_handler = stream.get_error_handler() + resolution = error_handler.interpret_response(InvalidURL()) + assert resolution.response_action == ResponseAction.FAIL + assert resolution.failure_type == FailureType.config_error + class PostHttpStream(StubBasicReadHttpStream): http_method = "POST" From 8a88e1dd4870bbfc11d6a8689405f2f249cd3668 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:15:39 +0000 Subject: [PATCH 15/18] style: Fix formatting with ruff Co-Authored-By: Aaron Steers --- airbyte_cdk/sources/streams/http/exceptions.py | 6 +++++- unit_tests/sources/streams/http/test_http.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/exceptions.py b/airbyte_cdk/sources/streams/http/exceptions.py index 40d3300ec..27e3f5d2c 100644 --- a/airbyte_cdk/sources/streams/http/exceptions.py +++ b/airbyte_cdk/sources/streams/http/exceptions.py @@ -75,4 +75,8 @@ def __init__( error_message: str = "", ): self.url = url - super().__init__(request=request, response=response, error_message=error_message or f"Failed to resolve DNS for URL: {url}") + super().__init__( + request=request, + response=response, + error_message=error_message or f"Failed to resolve DNS for URL: {url}", + ) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 5deaa9866..17b86a117 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -346,11 +346,14 @@ def test_dns_resolution_error_retry(): error_handler = stream.get_error_handler() request = requests.PreparedRequest() request.url = "https://example.com" - dns_error = DNSResolutionError(url="https://example.com", request=request, response=Exception("DNS lookup failed")) + dns_error = DNSResolutionError( + url="https://example.com", request=request, response=Exception("DNS lookup failed") + ) resolution = error_handler.interpret_response(dns_error) assert resolution.response_action == ResponseAction.RETRY assert resolution.failure_type == FailureType.transient_error + def test_invalid_url_fails(): """Test that invalid URLs fail immediately""" stream = StubHttpStreamWithErrorHandler() From 8506c83767d877070e4451d1089f7bc7435643f5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:21:21 +0000 Subject: [PATCH 16/18] fix: Add DNSResolutionError import Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 17b86a117..5a397b7d0 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -27,6 +27,7 @@ ) from airbyte_cdk.sources.streams.http.exceptions import ( DefaultBackoffException, + DNSResolutionError, RequestBodyException, UserDefinedBackoffException, ) From 57e35784fe443cdb6c41f5cb8018ea9e40aa6fce Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 20:57:12 +0000 Subject: [PATCH 17/18] style: Fix formatting in http_client.py Co-Authored-By: Aaron Steers --- airbyte_cdk/sources/streams/http/http_client.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index c4fa86866..22c43ef14 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -36,6 +36,7 @@ ) from airbyte_cdk.sources.streams.http.exceptions import ( DefaultBackoffException, + DNSResolutionError, RateLimitBackoffException, RequestBodyException, UserDefinedBackoffException, @@ -300,6 +301,16 @@ def _send( try: response = self._session.send(request, **request_kwargs) + except requests.ConnectionError as e: + if "Name or service not known" in str(e) or "nodename nor servname provided" in str(e): + assert ( + request.url is not None + ), "Request URL cannot be None for DNS resolution error" + exc = DNSResolutionError( + url=request.url, request=request, response=e, error_message=str(e) + ) + else: + exc = e except requests.RequestException as e: exc = e From 9148cd545da4d234832d7a4f84d3b8b878544b91 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 22:28:39 +0000 Subject: [PATCH 18/18] test: Add test for DNS resolution error from ConnectionError Co-Authored-By: Aaron Steers --- unit_tests/sources/streams/http/test_http.py | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index 5a397b7d0..7439eeaae 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -10,7 +10,7 @@ import pytest import requests -from requests.exceptions import InvalidURL +from requests.exceptions import ConnectionError, InvalidURL from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type from airbyte_cdk.sources.streams import CheckpointMixin @@ -364,6 +364,25 @@ def test_invalid_url_fails(): assert resolution.failure_type == FailureType.config_error +def test_dns_resolution_error_retry_with_connection_error(): + """Test that DNS resolution errors from ConnectionError are properly handled""" + stream = StubHttpStreamWithErrorHandler() + send_mock = MagicMock( + side_effect=requests.ConnectionError( + "HTTPSConnectionPool(host='api.example.com', port=443): " + "Max retries exceeded with url: /v1/data (Caused by " + 'NameResolutionError(": Failed to resolve 'api.example.com' " + '(Name or service not known)"))' + ) + ) + stream._http_client._session.send = send_mock + + with pytest.raises(DefaultBackoffException): + list(stream.read_records(SyncMode.full_refresh)) + assert send_mock.call_count == stream.max_retries + 1 + + class PostHttpStream(StubBasicReadHttpStream): http_method = "POST"