Merge pull request #154 from FreeshardBase/fix/clayde/reclaim-eagain-log

fix(memory_pressure): don't warn on expected EAGAIN from memory.reclaim

Author: max-tet

shard_core/service/memory_pressure.py

@@ -1,4 +1,5 @@
 import asyncio
+import errno
 import logging
 import re
 from pathlib import Path
@@ -50,9 +51,18 @@ def _reclaim_container(container_id: str):
     try:
         (cgroup / "memory.reclaim").write_text(f"{current}\n")
     except OSError as e:
-        # Partial reclaim surfaces as EAGAIN once nothing more can be paged
-        # out — the pages that could move are in swap already.
-        log.warning(f"memory.reclaim incomplete for container {container_id}: {e}")
+        # We request the container's full RSS, which can never be reclaimed in
+        # whole (some pages are always resident), so memory.reclaim returns
+        # EAGAIN on essentially every pause once it has paged out what it can —
+        # that is the expected terminal signal, not a failure. Only surface a
+        # warning for genuine errors (missing file, EPERM, ...).
+        if e.errno == errno.EAGAIN:
+            log.debug(
+                f"memory.reclaim reached EAGAIN for container {container_id} "
+                "(expected — paged out what it could)"
+            )
+        else:
+            log.warning(f"memory.reclaim failed for container {container_id}: {e}")
 
 
 def _find_cgroup(container_id: str) -> Path | None:

tests/test_memory_pressure.py

@@ -1,3 +1,6 @@
+import errno
+import logging
+from pathlib import Path
 from unittest.mock import AsyncMock, patch
 
 import pytest
@@ -67,6 +70,41 @@ def test_reclaim_container_zero_usage_writes_nothing(fake_cgroup_root):
     assert (cgroup / "memory.reclaim").read_text() == ""
 
 
+def test_reclaim_container_eagain_is_debug_not_warning(
+    fake_cgroup_root, caplog, monkeypatch
+):
+    # Requesting the full RSS always ends in EAGAIN once nothing more can be
+    # paged out — expected on every pause, so it must not log a warning.
+    _make_cgroup(fake_cgroup_root, "docker/eag123", 4096)
+
+    def _raise_eagain(self, *args, **kwargs):
+        raise OSError(errno.EAGAIN, "write could not complete without blocking")
+
+    monkeypatch.setattr(Path, "write_text", _raise_eagain)
+    with caplog.at_level(logging.DEBUG, logger=memory_pressure.log.name):
+        memory_pressure._reclaim_container("eag123")
+
+    assert not any(r.levelno == logging.WARNING for r in caplog.records)
+    assert any("EAGAIN" in r.getMessage() for r in caplog.records)
+
+
+def test_reclaim_container_other_oserror_warns(fake_cgroup_root, caplog, monkeypatch):
+    # A genuine error (not EAGAIN) still surfaces as a warning.
+    _make_cgroup(fake_cgroup_root, "docker/err123", 4096)
+
+    def _raise_eperm(self, *args, **kwargs):
+        raise OSError(errno.EPERM, "operation not permitted")
+
+    monkeypatch.setattr(Path, "write_text", _raise_eperm)
+    with caplog.at_level(logging.DEBUG, logger=memory_pressure.log.name):
+        memory_pressure._reclaim_container("err123")
+
+    assert any(
+        r.levelno == logging.WARNING and "failed" in r.getMessage()
+        for r in caplog.records
+    )
+
+
 async def test_reclaim_compose_stack_reclaims_each_container(fake_cgroup_root):
     cgroup_a = _make_cgroup(fake_cgroup_root, "docker/aaa", 100)
     cgroup_b = _make_cgroup(fake_cgroup_root, "docker/bbb", 200)