Skip to content

Commit 61c86fb

Browse files
committed
Retry targeted pickle cache entry removal
1 parent 19f9c58 commit 61c86fb

2 files changed

Lines changed: 60 additions & 14 deletions

File tree

src/cachier/cores/pickle.py

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -172,19 +172,22 @@ def _clear_all_cache_files(self) -> None:
172172
path, name = os.path.split(self.cache_fpath)
173173
for subpath in os.listdir(path):
174174
if subpath.startswith(f"{name}_"):
175-
fpath = os.path.join(path, subpath)
176-
# Retry loop to handle Windows mandatory file-locking (WinError 32):
177-
# portalocker holds an exclusive lock while a thread is computing,
178-
# so os.remove() may fail transiently until the lock is released.
179-
for attempt in range(3): # pragma: no branch
180-
try:
181-
os.remove(fpath)
182-
break
183-
except PermissionError:
184-
if attempt < 2:
185-
time.sleep(0.1 * (attempt + 1))
186-
else:
187-
raise
175+
self._remove_cache_file_with_retries(os.path.join(path, subpath))
176+
177+
@staticmethod
178+
def _remove_cache_file_with_retries(fpath: str) -> None:
179+
# Retry loop to handle Windows mandatory file-locking (WinError 32):
180+
# portalocker holds an exclusive lock while a thread is computing,
181+
# so os.remove() may fail transiently until the lock is released.
182+
for attempt in range(3): # pragma: no branch
183+
try:
184+
os.remove(fpath)
185+
break
186+
except PermissionError:
187+
if attempt < 2:
188+
time.sleep(0.1 * (attempt + 1))
189+
else:
190+
raise
188191

189192
def _clear_being_calculated_all_cache_files(self) -> None:
190193
path, name = os.path.split(self.cache_fpath)
@@ -423,7 +426,7 @@ def clear_cache(self) -> None:
423426
def clear_cache_entry(self, key: str) -> None:
424427
if self.separate_files:
425428
with suppress(FileNotFoundError):
426-
os.remove(f"{self.cache_fpath}_{key}")
429+
self._remove_cache_file_with_retries(f"{self.cache_fpath}_{key}")
427430
return
428431

429432
with self.lock:

tests/pickle_tests/test_pickle_core.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1445,6 +1445,49 @@ def flaky_remove(path):
14451445
assert not os.path.exists(dummy_file)
14461446

14471447

1448+
@pytest.mark.pickle
1449+
def test_clear_cache_entry_retries_on_permission_error(tmp_path):
1450+
"""Test clear_cache_entry retries on PermissionError then succeeds."""
1451+
core = _PickleCore(
1452+
hash_func=None,
1453+
cache_dir=tmp_path,
1454+
pickle_reload=False,
1455+
wait_for_calc_timeout=10,
1456+
separate_files=True,
1457+
)
1458+
1459+
def mock_func():
1460+
pass
1461+
1462+
core.set_func(mock_func)
1463+
1464+
cache_fpath = core.cache_fpath
1465+
dummy_file = cache_fpath + "_dummykey"
1466+
with open(dummy_file, "wb") as f:
1467+
f.write(b"")
1468+
1469+
real_remove = os.remove
1470+
call_count = 0
1471+
1472+
def flaky_remove(path):
1473+
nonlocal call_count
1474+
call_count += 1
1475+
if call_count < 3:
1476+
raise PermissionError("locked")
1477+
real_remove(path)
1478+
1479+
with (
1480+
patch("cachier.cores.pickle.os.remove", side_effect=flaky_remove),
1481+
patch("cachier.cores.pickle.time.sleep") as mock_sleep,
1482+
):
1483+
core.clear_cache_entry("dummykey")
1484+
assert mock_sleep.call_count == 2
1485+
mock_sleep.assert_any_call(0.1)
1486+
mock_sleep.assert_any_call(0.2)
1487+
1488+
assert not os.path.exists(dummy_file)
1489+
1490+
14481491
@pytest.mark.pickle
14491492
def test_clear_all_cache_files_raises_on_persistent_permission_error(tmp_path):
14501493
"""Test _clear_all_cache_files re-raises PermissionError after all retries."""

0 commit comments

Comments
 (0)