Skip to content

Commit cd9aa33

Browse files
pnilanpnilan
committed
fix: improve user-facing HTTP error messages with source attribution and stream context
Default HTTP error messages now clearly attribute errors to the source's API (not Airbyte platform), and HttpClient injects the stream name and HTTP status code for user-facing context. Example: "Stream 'users': HTTP 403. Source's API denied access. Configured credentials have insufficient permissions." Also fixes the default fallback connector error message to be more specific.
1 parent 7f41401 commit cd9aa33

12 files changed

Lines changed: 53 additions & 47 deletions

File tree

‎airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,56 +31,56 @@
3131
400: ErrorResolution(
3232
response_action=ResponseAction.FAIL,
3333
failure_type=FailureType.system_error,
34-
error_message="HTTP Status Code: 400. Error: Bad request. Please check your request parameters.",
34+
error_message="Bad request response from source's API.",
3535
),
3636
401: ErrorResolution(
3737
response_action=ResponseAction.FAIL,
3838
failure_type=FailureType.config_error,
39-
error_message="HTTP Status Code: 401. Error: Unauthorized. Please ensure you are authenticated correctly.",
39+
error_message="Authentication failed on source's API. Credentials may be invalid, expired, or lack required access.",
4040
),
4141
403: ErrorResolution(
4242
response_action=ResponseAction.FAIL,
4343
failure_type=FailureType.config_error,
44-
error_message="HTTP Status Code: 403. Error: Forbidden. You don't have permission to access this resource.",
44+
error_message="Source's API denied access. Configured credentials have insufficient permissions.",
4545
),
4646
404: ErrorResolution(
4747
response_action=ResponseAction.FAIL,
4848
failure_type=FailureType.system_error,
49-
error_message="HTTP Status Code: 404. Error: Not found. The requested resource was not found on the server.",
49+
error_message="Requested resource not found on source's API.",
5050
),
5151
405: ErrorResolution(
5252
response_action=ResponseAction.FAIL,
5353
failure_type=FailureType.system_error,
54-
error_message="HTTP Status Code: 405. Error: Method not allowed. Please check your request method.",
54+
error_message="Method not allowed by source's API.",
5555
),
5656
408: ErrorResolution(
5757
response_action=ResponseAction.RETRY,
5858
failure_type=FailureType.transient_error,
59-
error_message="HTTP Status Code: 408. Error: Request timeout.",
59+
error_message="Request to source's API timed out.",
6060
),
6161
429: ErrorResolution(
6262
response_action=ResponseAction.RATE_LIMITED,
6363
failure_type=FailureType.transient_error,
64-
error_message="HTTP Status Code: 429. Error: Too many requests.",
64+
error_message="Rate limit exceeded on source's API.",
6565
),
6666
500: ErrorResolution(
6767
response_action=ResponseAction.RETRY,
6868
failure_type=FailureType.transient_error,
69-
error_message="HTTP Status Code: 500. Error: Internal server error.",
69+
error_message="Internal server error from source's API.",
7070
),
7171
502: ErrorResolution(
7272
response_action=ResponseAction.RETRY,
7373
failure_type=FailureType.transient_error,
74-
error_message="HTTP Status Code: 502. Error: Bad gateway.",
74+
error_message="Bad gateway response from source's API.",
7575
),
7676
503: ErrorResolution(
7777
response_action=ResponseAction.RETRY,
7878
failure_type=FailureType.transient_error,
79-
error_message="HTTP Status Code: 503. Error: Service unavailable.",
79+
error_message="Source's API is temporarily unavailable.",
8080
),
8181
504: ErrorResolution(
8282
response_action=ResponseAction.RETRY,
8383
failure_type=FailureType.transient_error,
84-
error_message="HTTP Status Code: 504. Error: Gateway timeout.",
84+
error_message="Gateway timeout from source's API.",
8585
),
8686
}

‎airbyte_cdk/sources/streams/http/http_client.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@ def _send_with_retry(
313313
self._logger.error(f"Retries exhausted with backoff exception.", exc_info=True)
314314
raise MessageRepresentationAirbyteTracedErrors(
315315
internal_message=f"Exhausted available request attempts. Exception: {e}",
316-
message=f"Exhausted available request attempts. Please see logs for more details. Exception: {e}",
316+
message=f"Request retry limit exhausted. See logs for details.",
317317
failure_type=e.failure_type or FailureType.system_error,
318318
exception=e,
319319
stream_descriptor=StreamDescriptor(name=self._name),
@@ -424,6 +424,13 @@ def _evict_key(self, prepared_request: requests.PreparedRequest) -> None:
424424
if prepared_request in self._request_attempt_count:
425425
del self._request_attempt_count[prepared_request]
426426

427+
def _format_error_message(self, error_resolution: ErrorResolution, response: Optional[requests.Response]) -> Optional[str]:
428+
"""Prepend stream name and HTTP status code to the error resolution message for user-facing context."""
429+
if not error_resolution.error_message:
430+
return None
431+
status_prefix = f"HTTP {response.status_code}. " if response is not None else ""
432+
return f"Stream '{self._name}': {status_prefix}{error_resolution.error_message}"
433+
427434
def _handle_error_resolution(
428435
self,
429436
response: Optional[requests.Response],
@@ -497,7 +504,7 @@ def _handle_error_resolution(
497504

498505
raise MessageRepresentationAirbyteTracedErrors(
499506
internal_message=error_message,
500-
message=error_resolution.error_message or error_message,
507+
message=self._format_error_message(error_resolution, response) or error_message,
501508
failure_type=error_resolution.failure_type,
502509
)
503510

@@ -507,7 +514,7 @@ def _handle_error_resolution(
507514
else:
508515
log_message = f"Ignoring response for '{request.method}' request to '{request.url}' with error '{exc}'"
509516

510-
self._logger.info(error_resolution.error_message or log_message)
517+
self._logger.info(self._format_error_message(error_resolution, response) or log_message)
511518

512519
# TODO: Consider dynamic retry count depending on subsequent error codes
513520
elif error_resolution.response_action in (
@@ -525,7 +532,7 @@ def _handle_error_resolution(
525532
user_defined_backoff_time = backoff_time
526533
break
527534
error_message = (
528-
error_resolution.error_message
535+
self._format_error_message(error_resolution, response)
529536
or f"Request to {request.url} failed with failure type {error_resolution.failure_type}, response action {error_resolution.response_action}."
530537
)
531538

‎airbyte_cdk/utils/traced_exception.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ def as_airbyte_message(
6868
emitted_at=now_millis,
6969
error=AirbyteErrorTraceMessage(
7070
message=self.message
71-
or "Something went wrong in the connector. See the logs for more details.",
71+
or "Unhandled connector error. See logs for details.",
7272
internal_message=self.internal_message,
7373
failure_type=self.failure_type,
7474
stack_trace=stack_trace_str,

‎unit_tests/sources/declarative/checks/test_check_dynamic_stream.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,14 +112,14 @@
112112
404,
113113
Status.FAILED,
114114
True,
115-
["Not found. The requested resource was not found on the server."],
115+
["Requested resource not found on source's API."],
116116
id="test_stream_unavailable_unhandled_error",
117117
),
118118
pytest.param(
119119
403,
120120
Status.FAILED,
121121
True,
122-
["Forbidden. You don't have permission to access this resource."],
122+
["Source's API denied access. Configured credentials have insufficient permissions."],
123123
id="test_stream_unavailable_handled_error",
124124
),
125125
pytest.param(200, Status.SUCCEEDED, True, [], id="test_stream_available"),
@@ -128,7 +128,7 @@
128128
401,
129129
Status.FAILED,
130130
True,
131-
["Unauthorized. Please ensure you are authenticated correctly."],
131+
["Authentication failed on source's API. Credentials may be invalid, expired, or lack required access."],
132132
id="test_stream_unauthorized_error",
133133
),
134134
],

‎unit_tests/sources/declarative/checks/test_check_stream.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -113,13 +113,13 @@ def test_check_stream_with_no_stream_slices_aborts():
113113
"test_stream_unavailable_unhandled_error",
114114
404,
115115
False,
116-
["Not found. The requested resource was not found on the server."],
116+
["Requested resource not found on source's API."],
117117
),
118118
(
119119
"test_stream_unavailable_handled_error",
120120
403,
121121
False,
122-
["Forbidden. You don't have permission to access this resource."],
122+
["Source's API denied access. Configured credentials have insufficient permissions."],
123123
),
124124
("test_stream_available", 200, True, []),
125125
],
@@ -521,7 +521,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
521521
Status.FAILED,
522522
False,
523523
404,
524-
["Not found. The requested resource was not found on the server."],
524+
["Requested resource not found on source's API."],
525525
0,
526526
id="test_stream_unavailable_unhandled_error",
527527
),
@@ -530,7 +530,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
530530
Status.FAILED,
531531
False,
532532
403,
533-
["Forbidden. You don't have permission to access this resource."],
533+
["Source's API denied access. Configured credentials have insufficient permissions."],
534534
0,
535535
id="test_stream_unavailable_handled_error",
536536
),
@@ -539,7 +539,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
539539
Status.FAILED,
540540
False,
541541
401,
542-
["Unauthorized. Please ensure you are authenticated correctly."],
542+
["Authentication failed on source's API. Credentials may be invalid, expired, or lack required access."],
543543
0,
544544
id="test_stream_unauthorized_error",
545545
),
@@ -563,7 +563,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
563563
Status.FAILED,
564564
False,
565565
404,
566-
["Not found. The requested resource was not found on the server."],
566+
["Requested resource not found on source's API."],
567567
0,
568568
id="test_dynamic_stream_unavailable_unhandled_error",
569569
),
@@ -587,7 +587,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
587587
Status.FAILED,
588588
False,
589589
403,
590-
["Forbidden. You don't have permission to access this resource."],
590+
["Source's API denied access. Configured credentials have insufficient permissions."],
591591
0,
592592
id="test_dynamic_stream_unavailable_handled_error",
593593
),
@@ -611,7 +611,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
611611
Status.FAILED,
612612
False,
613613
401,
614-
["Unauthorized. Please ensure you are authenticated correctly."],
614+
["Authentication failed on source's API. Credentials may be invalid, expired, or lack required access."],
615615
0,
616616
id="test_dynamic_stream_unauthorized_error",
617617
),

‎unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ def test_default_error_handler_with_default_response_filter(
9292
),
9393
ResponseAction.RETRY,
9494
FailureType.system_error,
95-
"HTTP Status Code: 400. Error: Bad request. Please check your request parameters.",
95+
"Bad request response from source's API.",
9696
),
9797
(
9898
"_with_http_response_status_402_fail_with_default_failure_type",
@@ -118,7 +118,7 @@ def test_default_error_handler_with_default_response_filter(
118118
),
119119
ResponseAction.FAIL,
120120
FailureType.config_error,
121-
"HTTP Status Code: 403. Error: Forbidden. You don't have permission to access this resource.",
121+
"Source's API denied access. Configured credentials have insufficient permissions.",
122122
),
123123
(
124124
"_with_http_response_status_200_fail_with_contained_error_message",

‎unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
ErrorResolution(
4545
response_action=ResponseAction.IGNORE,
4646
failure_type=FailureType.config_error,
47-
error_message="HTTP Status Code: 403. Error: Forbidden. You don't have permission to access this resource.",
47+
error_message="Source's API denied access. Configured credentials have insufficient permissions.",
4848
),
4949
id="test_http_code_matches_ignore_action",
5050
),
@@ -59,7 +59,7 @@
5959
ErrorResolution(
6060
response_action=ResponseAction.RETRY,
6161
failure_type=FailureType.transient_error,
62-
error_message="HTTP Status Code: 429. Error: Too many requests.",
62+
error_message="Rate limit exceeded on source's API.",
6363
),
6464
id="test_http_code_matches_retry_action",
6565
),
@@ -104,7 +104,7 @@
104104
ErrorResolution(
105105
response_action=ResponseAction.FAIL,
106106
failure_type=FailureType.config_error,
107-
error_message="HTTP Status Code: 403. Error: Forbidden. You don't have permission to access this resource.",
107+
error_message="Source's API denied access. Configured credentials have insufficient permissions.",
108108
),
109109
id="test_predicate_matches_headers",
110110
),

‎unit_tests/sources/streams/http/error_handlers/test_http_status_error_handler.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,13 @@ def test_given_ok_response_http_status_error_handler_returns_success_action(mock
3434
403,
3535
ResponseAction.FAIL,
3636
FailureType.config_error,
37-
"HTTP Status Code: 403. Error: Forbidden. You don't have permission to access this resource.",
37+
"Source's API denied access. Configured credentials have insufficient permissions.",
3838
),
3939
(
4040
404,
4141
ResponseAction.FAIL,
4242
FailureType.system_error,
43-
"HTTP Status Code: 404. Error: Not found. The requested resource was not found on the server.",
43+
"Requested resource not found on source's API.",
4444
),
4545
],
4646
)

‎unit_tests/sources/streams/http/test_availability_strategy.py‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@ def retry_factor(self) -> float:
4949
{"error": "Something went wrong"},
5050
False,
5151
[
52-
"Forbidden. You don't have permission to access this resource.",
53-
"Forbidden. You don't have permission to access this resource.",
52+
"Source's API denied access. Configured credentials have insufficient permissions.",
53+
"Source's API denied access. Configured credentials have insufficient permissions.",
5454
],
5555
),
5656
(200, {}, True, []),
@@ -59,8 +59,8 @@ def retry_factor(self) -> float:
5959
@pytest.mark.parametrize(
6060
("include_source", "expected_docs_url_messages"),
6161
[
62-
(True, ["Forbidden. You don't have permission to access this resource."]),
63-
(False, ["Forbidden. You don't have permission to access this resource."]),
62+
(True, ["Source's API denied access. Configured credentials have insufficient permissions."]),
63+
(False, ["Source's API denied access. Configured credentials have insufficient permissions."]),
6464
],
6565
)
6666
@pytest.mark.parametrize("records_as_list", [True, False])
@@ -105,10 +105,9 @@ def test_http_availability_raises_unhandled_error(mocker):
105105
req.status_code = 404
106106
mocker.patch.object(requests.Session, "send", return_value=req)
107107

108-
assert (
109-
False,
110-
"HTTP Status Code: 404. Error: Not found. The requested resource was not found on the server.",
111-
) == HttpAvailabilityStrategy().check_availability(http_stream, logger)
108+
is_available, reason = HttpAvailabilityStrategy().check_availability(http_stream, logger)
109+
assert is_available is False
110+
assert "Requested resource not found on source's API." in reason
112111

113112

114113
def test_send_handles_retries_when_checking_availability(mocker, caplog):

‎unit_tests/sources/streams/http/test_http.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ def get_error_handler(self) -> Optional[ErrorHandler]:
222222
send_mock = mocker.patch.object(requests.Session, "send", return_value=req)
223223

224224
with pytest.raises(
225-
AirbyteTracedException, match="Exception: HTTP Status Code: 429. Error: Too many requests."
225+
AirbyteTracedException, match="Rate limit exceeded on source's API."
226226
):
227227
list(stream.read_records(SyncMode.full_refresh))
228228
if retries <= 0:
@@ -316,7 +316,7 @@ def test_raise_on_http_errors_off_429(mocker):
316316
mocker.patch.object(requests.Session, "send", return_value=req)
317317
with pytest.raises(
318318
AirbyteTracedException,
319-
match="Exhausted available request attempts. Please see logs for more details. Exception: HTTP Status Code: 429. Error: Too many requests.",
319+
match="Request retry limit exhausted. See logs for details.",
320320
):
321321
stream.exit_on_rate_limit = True
322322
list(stream.read_records(SyncMode.full_refresh))

0 commit comments

Comments
 (0)