Skip to content

Commit 8a40f9d

Browse files
potiukpotiuk
committed
Retry uv pip install on cargo-related transient failures
1 parent 860277d commit 8a40f9d

4 files changed

Lines changed: 180 additions & 61 deletions

File tree

‎Dockerfile‎

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1195,6 +1195,40 @@ COPY <<"EOF" /install_airflow_when_building_images.sh
11951195

11961196
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
11971197

1198+
function run_uv_with_cargo_retry() {
1199+
local max_attempts=3
1200+
local sleep_seconds=30
1201+
local attempt
1202+
local exit_code
1203+
local output_file
1204+
output_file=$(mktemp)
1205+
# shellcheck disable=SC2064
1206+
trap "rm -f ${output_file}" RETURN
1207+
for attempt in $(seq 1 ${max_attempts}); do
1208+
echo "${COLOR_BLUE}Attempt ${attempt} of ${max_attempts} for: $*${COLOR_RESET}"
1209+
set +e
1210+
set -x
1211+
"$@" 2>&1 | tee "${output_file}"
1212+
exit_code=${PIPESTATUS[0]}
1213+
set +x
1214+
set -e
1215+
if [[ ${exit_code} == 0 ]]; then
1216+
return 0
1217+
fi
1218+
if ! grep -qi -e "cargo" -e "maturin" "${output_file}"; then
1219+
echo "${COLOR_YELLOW}Failure is not cargo-related, not retrying.${COLOR_RESET}"
1220+
return ${exit_code}
1221+
fi
1222+
if [[ ${attempt} -lt ${max_attempts} ]]; then
1223+
echo "${COLOR_YELLOW}Attempt ${attempt} failed with cargo-related error. Sleeping ${sleep_seconds}s before retry...${COLOR_RESET}"
1224+
sleep ${sleep_seconds}
1225+
else
1226+
echo "${COLOR_RED}All ${max_attempts} attempts failed.${COLOR_RESET}"
1227+
fi
1228+
done
1229+
return ${exit_code}
1230+
}
1231+
11981232
function install_from_sources() {
11991233
local extra_sync_flags
12001234
extra_sync_flags=""
@@ -1216,24 +1250,20 @@ function install_from_sources() {
12161250
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
12171251
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
12181252
# See https://bugs.launchpad.net/lxml/+bug/2110068
1219-
set -x
1220-
uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
1253+
run_uv_with_cargo_retry uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
12211254
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
12221255
--no-python-downloads --no-managed-python
12231256
else
1224-
set +x
12251257
echo
12261258
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
12271259
echo
12281260
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
12291261
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
12301262
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
12311263
# See https://bugs.launchpad.net/lxml/+bug/2110068
1232-
set -x
1233-
if ! uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
1264+
if ! run_uv_with_cargo_retry uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
12341265
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
12351266
--no-python-downloads --no-managed-python; then
1236-
set +x
12371267
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
12381268
echo
12391269
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
@@ -1245,11 +1275,9 @@ function install_from_sources() {
12451275
echo
12461276
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv sync without --frozen).${COLOR_RESET}"
12471277
echo
1248-
set -x
1249-
uv sync --all-packages --group dev --group docs --group docs-gen \
1278+
run_uv_with_cargo_retry uv sync --all-packages --group dev --group docs --group docs-gen \
12501279
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
12511280
--no-python-downloads --no-managed-python
1252-
set +x
12531281
fi
12541282
fi
12551283
}
@@ -1276,16 +1304,12 @@ function install_from_external_spec() {
12761304
echo
12771305
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
12781306
echo
1279-
set -x
1280-
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
1281-
set +x
1307+
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
12821308
else
12831309
echo
12841310
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
12851311
echo
1286-
set -x
1287-
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
1288-
set +x
1312+
if ! run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
12891313
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
12901314
echo
12911315
echo "${COLOR_RED}Failing because constraints installation failed and fallback is disabled.${COLOR_RESET}"
@@ -1297,9 +1321,7 @@ function install_from_external_spec() {
12971321
echo
12981322
echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}"
12991323
echo
1300-
set -x
1301-
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
1302-
set +x
1324+
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
13031325
fi
13041326
fi
13051327
}

‎Dockerfile.ci‎

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -900,6 +900,40 @@ COPY <<"EOF" /install_airflow_when_building_images.sh
900900

901901
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
902902

903+
function run_uv_with_cargo_retry() {
904+
local max_attempts=3
905+
local sleep_seconds=30
906+
local attempt
907+
local exit_code
908+
local output_file
909+
output_file=$(mktemp)
910+
# shellcheck disable=SC2064
911+
trap "rm -f ${output_file}" RETURN
912+
for attempt in $(seq 1 ${max_attempts}); do
913+
echo "${COLOR_BLUE}Attempt ${attempt} of ${max_attempts} for: $*${COLOR_RESET}"
914+
set +e
915+
set -x
916+
"$@" 2>&1 | tee "${output_file}"
917+
exit_code=${PIPESTATUS[0]}
918+
set +x
919+
set -e
920+
if [[ ${exit_code} == 0 ]]; then
921+
return 0
922+
fi
923+
if ! grep -qi -e "cargo" -e "maturin" "${output_file}"; then
924+
echo "${COLOR_YELLOW}Failure is not cargo-related, not retrying.${COLOR_RESET}"
925+
return ${exit_code}
926+
fi
927+
if [[ ${attempt} -lt ${max_attempts} ]]; then
928+
echo "${COLOR_YELLOW}Attempt ${attempt} failed with cargo-related error. Sleeping ${sleep_seconds}s before retry...${COLOR_RESET}"
929+
sleep ${sleep_seconds}
930+
else
931+
echo "${COLOR_RED}All ${max_attempts} attempts failed.${COLOR_RESET}"
932+
fi
933+
done
934+
return ${exit_code}
935+
}
936+
903937
function install_from_sources() {
904938
local extra_sync_flags
905939
extra_sync_flags=""
@@ -921,24 +955,20 @@ function install_from_sources() {
921955
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
922956
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
923957
# See https://bugs.launchpad.net/lxml/+bug/2110068
924-
set -x
925-
uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
958+
run_uv_with_cargo_retry uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
926959
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
927960
--no-python-downloads --no-managed-python
928961
else
929-
set +x
930962
echo
931963
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
932964
echo
933965
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
934966
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
935967
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
936968
# See https://bugs.launchpad.net/lxml/+bug/2110068
937-
set -x
938-
if ! uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
969+
if ! run_uv_with_cargo_retry uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
939970
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
940971
--no-python-downloads --no-managed-python; then
941-
set +x
942972
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
943973
echo
944974
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
@@ -950,11 +980,9 @@ function install_from_sources() {
950980
echo
951981
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv sync without --frozen).${COLOR_RESET}"
952982
echo
953-
set -x
954-
uv sync --all-packages --group dev --group docs --group docs-gen \
983+
run_uv_with_cargo_retry uv sync --all-packages --group dev --group docs --group docs-gen \
955984
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
956985
--no-python-downloads --no-managed-python
957-
set +x
958986
fi
959987
fi
960988
}
@@ -981,16 +1009,12 @@ function install_from_external_spec() {
9811009
echo
9821010
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
9831011
echo
984-
set -x
985-
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
986-
set +x
1012+
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
9871013
else
9881014
echo
9891015
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
9901016
echo
991-
set -x
992-
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
993-
set +x
1017+
if ! run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
9941018
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
9951019
echo
9961020
echo "${COLOR_RED}Failing because constraints installation failed and fallback is disabled.${COLOR_RESET}"
@@ -1002,9 +1026,7 @@ function install_from_external_spec() {
10021026
echo
10031027
echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}"
10041028
echo
1005-
set -x
1006-
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
1007-
set +x
1029+
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
10081030
fi
10091031
fi
10101032
}

‎scripts/docker/install_airflow_when_building_images.sh‎

Lines changed: 45 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,45 @@
3535
# shellcheck source=scripts/docker/common.sh
3636
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
3737

38+
# Retry wrapper for uv commands that may fail with transient cargo/maturin build errors.
39+
# Workaround for https://github.com/astral-sh/uv/issues/18801
40+
# Usage: run_uv_with_cargo_retry <command...>
41+
# Returns the exit code of the command (non-zero only after all retries exhausted for cargo errors,
42+
# or immediately for non-cargo failures).
43+
function run_uv_with_cargo_retry() {
44+
local max_attempts=3
45+
local sleep_seconds=30
46+
local attempt
47+
local exit_code
48+
local output_file
49+
output_file=$(mktemp)
50+
# shellcheck disable=SC2064
51+
trap "rm -f ${output_file}" RETURN
52+
for attempt in $(seq 1 ${max_attempts}); do
53+
echo "${COLOR_BLUE}Attempt ${attempt} of ${max_attempts} for: $*${COLOR_RESET}"
54+
set +e
55+
set -x
56+
"$@" 2>&1 | tee "${output_file}"
57+
exit_code=${PIPESTATUS[0]}
58+
set +x
59+
set -e
60+
if [[ ${exit_code} == 0 ]]; then
61+
return 0
62+
fi
63+
if ! grep -qi -e "cargo" -e "maturin" "${output_file}"; then
64+
echo "${COLOR_YELLOW}Failure is not cargo-related, not retrying.${COLOR_RESET}"
65+
return ${exit_code}
66+
fi
67+
if [[ ${attempt} -lt ${max_attempts} ]]; then
68+
echo "${COLOR_YELLOW}Attempt ${attempt} failed with cargo-related error. Sleeping ${sleep_seconds}s before retry...${COLOR_RESET}"
69+
sleep ${sleep_seconds}
70+
else
71+
echo "${COLOR_RED}All ${max_attempts} attempts failed.${COLOR_RESET}"
72+
fi
73+
done
74+
return ${exit_code}
75+
}
76+
3877
function install_from_sources() {
3978
local extra_sync_flags
4079
extra_sync_flags=""
@@ -56,24 +95,20 @@ function install_from_sources() {
5695
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
5796
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
5897
# See https://bugs.launchpad.net/lxml/+bug/2110068
59-
set -x
60-
uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
98+
run_uv_with_cargo_retry uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \
6199
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
62100
--no-python-downloads --no-managed-python
63101
else
64-
set +x
65102
echo
66103
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
67104
echo
68105
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
69106
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
70107
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
71108
# See https://bugs.launchpad.net/lxml/+bug/2110068
72-
set -x
73-
if ! uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
109+
if ! run_uv_with_cargo_retry uv sync --all-packages --frozen --group dev --group docs --group docs-gen \
74110
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
75111
--no-python-downloads --no-managed-python; then
76-
set +x
77112
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
78113
echo
79114
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
@@ -85,11 +120,9 @@ function install_from_sources() {
85120
echo
86121
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv sync without --frozen).${COLOR_RESET}"
87122
echo
88-
set -x
89-
uv sync --all-packages --group dev --group docs --group docs-gen \
123+
run_uv_with_cargo_retry uv sync --all-packages --group dev --group docs --group docs-gen \
90124
--group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
91125
--no-python-downloads --no-managed-python
92-
set +x
93126
fi
94127
fi
95128
}
@@ -116,16 +149,12 @@ function install_from_external_spec() {
116149
echo
117150
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
118151
echo
119-
set -x
120-
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
121-
set +x
152+
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
122153
else
123154
echo
124155
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
125156
echo
126-
set -x
127-
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
128-
set +x
157+
if ! run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
129158
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
130159
echo
131160
echo "${COLOR_RED}Failing because constraints installation failed and fallback is disabled.${COLOR_RESET}"
@@ -137,9 +166,7 @@ function install_from_external_spec() {
137166
echo
138167
echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}"
139168
echo
140-
set -x
141-
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
142-
set +x
169+
run_uv_with_cargo_retry ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
143170
fi
144171
fi
145172
}

0 commit comments

Comments
 (0)