Skip to content

Commit c614e30

Browse files
fix(logger): Do not retry on 413s (#427)
fixes #415 supercedes #416 --------- Co-authored-by: david.lustig <david.lustig@klaviyo.com>
1 parent cc3f490 commit c614e30

2 files changed

Lines changed: 70 additions & 15 deletions

File tree

py/src/braintrust/logger.py

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1384,29 +1384,30 @@ def _submit_logs_request(self, items: Sequence[LogItemWithMeta], max_request_siz
13841384
if overflow_rows:
13851385
self._overflow_upload_count += 1
13861386
return
1387-
if error is None and resp is not None:
1388-
resp_errmsg = f"{resp.status_code}: {resp.text}"
1389-
else:
1390-
resp_errmsg = str(error)
1387+
has_response = error is None and resp is not None
1388+
is_413 = has_response and resp.status_code == 413
1389+
resp_errmsg = f"{resp.status_code}: {resp.text}" if has_response else str(error)
13911390

1392-
is_retrying = i + 1 < self.num_tries
1393-
retrying_text = "" if is_retrying else " Retrying"
1394-
errmsg = f"log request failed. Elapsed time: {time.time() - start_time} seconds. Payload size: {payload_bytes}.{retrying_text} Error: {resp_errmsg}"
1391+
should_retry = i + 1 < self.num_tries and not is_413
13951392

1396-
if not is_retrying and self.failed_publish_payloads_dir:
1393+
if not should_retry and self.failed_publish_payloads_dir:
13971394
_HTTPBackgroundLogger._write_payload_to_dir(
13981395
payload_dir=self.failed_publish_payloads_dir, payload=dataStr
13991396
)
14001397
self._log_failed_payloads_dir()
14011398

1402-
if not is_retrying and self.sync_flush:
1399+
retrying_text = " Retrying" if should_retry else ""
1400+
errmsg = f"log request failed. Elapsed time: {time.time() - start_time} seconds. Payload size: {payload_bytes}.{retrying_text} Error: {resp_errmsg}"
1401+
if not should_retry and self.sync_flush:
14031402
raise Exception(errmsg)
1404-
else:
1405-
print(errmsg, file=self.outfile)
1406-
if is_retrying:
1407-
sleep_time_s = BACKGROUND_LOGGER_BASE_SLEEP_TIME_S * (2**i)
1408-
print(f"Sleeping for {sleep_time_s}s", file=self.outfile)
1409-
time.sleep(sleep_time_s)
1403+
print(errmsg, file=self.outfile)
1404+
1405+
if is_413:
1406+
return
1407+
if should_retry:
1408+
sleep_time_s = BACKGROUND_LOGGER_BASE_SLEEP_TIME_S * (2**i)
1409+
print(f"Sleeping for {sleep_time_s}s", file=self.outfile)
1410+
time.sleep(sleep_time_s)
14101411

14111412
print(f"log request failed after {self.num_tries} retries. Dropping batch", file=self.outfile)
14121413

py/src/braintrust/test_logger.py

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,60 @@ def test_init_with_saved_parameters_attaches_reference(self):
187187
assert payload["parameters_version"] == "v1"
188188

189189

190+
class TestHTTPBackgroundLoggerLogs3(TestCase):
191+
def test_submit_logs_request_413_skips_retries(self) -> None:
192+
"""Any 413 while publishing ``/logs3`` cannot succeed on retry with the same payload.
193+
194+
``sync_flush`` controls whether the terminal failure raises instead of printing.
195+
"""
196+
from braintrust.logger import (
197+
LogItemWithMeta,
198+
Logs3OverflowInputRow,
199+
_HTTPBackgroundLogger,
200+
)
201+
202+
item = LogItemWithMeta(
203+
str_value="{}",
204+
overflow_meta=Logs3OverflowInputRow(
205+
object_ids={},
206+
has_comment=False,
207+
is_delete=False,
208+
byte_size=2,
209+
),
210+
)
211+
max_result = {"max_request_size": 10**9, "can_use_overflow": True}
212+
213+
for response_text in ("Request Too Long", "", "Payload Too Large"):
214+
for sync_flush in (False, True):
215+
with self.subTest(response_text=response_text, sync_flush=sync_flush):
216+
mock_resp = MagicMock()
217+
mock_resp.ok = False
218+
mock_resp.status_code = 413
219+
mock_resp.text = response_text
220+
221+
mock_conn = MagicMock()
222+
mock_conn.post.return_value = mock_resp
223+
224+
bg = _HTTPBackgroundLogger(LazyValue(lambda: mock_conn, use_mutex=False))
225+
bg.num_tries = 5
226+
bg.sync_flush = sync_flush
227+
bg.failed_publish_payloads_dir = "/tmp/failed-payloads"
228+
229+
with patch.object(_HTTPBackgroundLogger, "_write_payload_to_dir") as mock_write_payload:
230+
if sync_flush:
231+
with self.assertRaises(Exception) as cm:
232+
bg._submit_logs_request([item], max_result)
233+
self.assertIn("413", str(cm.exception))
234+
else:
235+
bg._submit_logs_request([item], max_result)
236+
237+
self.assertEqual(mock_conn.post.call_count, 1)
238+
mock_write_payload.assert_called_once()
239+
self.assertEqual(
240+
mock_write_payload.call_args.kwargs["payload_dir"], bg.failed_publish_payloads_dir
241+
)
242+
243+
190244
class TestLogger(TestCase):
191245
def test_load_prompt_prefers_version_over_environment_for_project_slug(self):
192246
mock_api_conn = MagicMock()

0 commit comments

Comments
 (0)