Skip to content

Commit db3a583

Browse files
fix(memory-monitor): use FailureType.transient_error for OOM fail-fast
OOM pressure near the cliff is a transient condition (workload/slice shape dependent) rather than an internal CDK logic error, so a retry with fresh process memory is meaningful. Flip failure_type from system_error to transient_error so orchestrators can retry appropriately. Co-Authored-By: patrick.nilan@airbyte.io <patrick.nilan@airbyte.io>
1 parent 5a054f8 commit db3a583

2 files changed

Lines changed: 3 additions & 3 deletions

File tree

airbyte_cdk/utils/memory_monitor.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ class MemoryMonitor:
126126
(default 5000) to 100 messages to narrow the race window near OOM.
127127
128128
**Fail-fast:** Raises ``AirbyteTracedException`` with
129-
``FailureType.system_error`` when *both*:
129+
``FailureType.transient_error`` when *both*:
130130
131131
1. Cgroup usage >= 95% of the container limit (container is near OOM-kill)
132132
2. Anonymous memory >= 85% of *current cgroup usage* (most of the charged
@@ -296,7 +296,7 @@ def check_memory_usage(self) -> None:
296296
f"Thresholds: cgroup >= {int(_CRITICAL_THRESHOLD * 100)}%, "
297297
f"anon share of usage >= {int(_ANON_SHARE_OF_USAGE_THRESHOLD * 100)}%."
298298
),
299-
failure_type=FailureType.system_error,
299+
failure_type=FailureType.transient_error,
300300
)
301301
elif not self._critical_logged:
302302
self._critical_logged = True

unit_tests/utils/test_memory_monitor.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ def test_raises_when_cgroup_critical_and_anon_share_of_usage_above_threshold() -
387387
):
388388
with pytest.raises(AirbyteTracedException) as exc_info:
389389
monitor.check_memory_usage()
390-
assert exc_info.value.failure_type == FailureType.system_error
390+
assert exc_info.value.failure_type == FailureType.transient_error
391391
assert "critical threshold" in (exc_info.value.message or "")
392392
assert "96%" in (exc_info.value.message or "")
393393
assert "anon share of usage" in (exc_info.value.internal_message or "")

0 commit comments

Comments
 (0)