From 3fa554dd9e401c6becbfed56dbede07db1a3548c Mon Sep 17 00:00:00 2001 From: Dan Braun Date: Fri, 12 Jun 2026 12:44:46 +0000 Subject: [PATCH] fix(lm): retry HTTP 408 from the HF CDN in the hub retry backend Job 703812 died at dataloader setup when a ranged parquet read got a 408 (Request Time-out) from the HF CDN. The #557 retry backend was active but its status_forcelist only covered 429/5xx, so urllib3 returned the 408 unretried and hf_raise_for_status raised, tearing down all 16 ranks. 408 is the same transient-timeout class that backend exists for, and only idempotent methods are retried, so adding it is safe. Co-Authored-By: Claude Fable 5 --- param_decomp_lab/infra/hf_http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/param_decomp_lab/infra/hf_http.py b/param_decomp_lab/infra/hf_http.py index 5d461e829..ba70620d5 100644 --- a/param_decomp_lab/infra/hf_http.py +++ b/param_decomp_lab/infra/hf_http.py @@ -4,7 +4,7 @@ adapter for metadata calls like `HfApi.repo_info` — the call `datasets` makes to resolve a streaming dataset's layout at startup. A single `ReadTimeout` there raises, and in a DDP job that one rank's failure tears down every rank before training begins. This mounts -a retrying adapter on the session factory so connect/read timeouts and 5xx/429 are +a retrying adapter on the session factory so connect/read timeouts and 408/429/5xx are retried with jittered backoff across *all* Hub HTTP calls (dataset, tokenizer, model). """ @@ -36,7 +36,7 @@ def configure_hf_http_retries(*, total_retries: int = 5, backoff_factor: float = status=total_retries, backoff_factor=backoff_factor, backoff_jitter=1.0, - status_forcelist=(429, 500, 502, 503, 504), + status_forcelist=(408, 429, 500, 502, 503, 504), allowed_methods=frozenset({"GET", "HEAD", "OPTIONS"}), respect_retry_after_header=True, raise_on_status=False,