Skip to content

Commit 5e35792

Browse files
committed
refactor(disk-check): use percentage-based disk space validation
- Changed from static 1GB minimum to percentage-based check - Default: 10% of total disk must be free (scales with disk size) - Configurable via RUNPOD_MIN_DISK_PERCENT environment variable - Benefits: - Small disks (50GB): ~5GB free required - Large disks (1TB): ~100GB free required - Automatically scales to machine resources - Updated tests to verify percentage-based logic
1 parent 1f7e83d commit 5e35792

2 files changed

Lines changed: 17 additions & 10 deletions

File tree

runpod/serverless/modules/rp_system_fitness.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626

2727
# Configuration via environment variables
2828
MIN_MEMORY_GB = float(os.environ.get("RUNPOD_MIN_MEMORY_GB", "4.0"))
29-
MIN_DISK_GB = float(os.environ.get("RUNPOD_MIN_DISK_GB", "1.0"))
29+
MIN_DISK_PERCENT = float(os.environ.get("RUNPOD_MIN_DISK_PERCENT", "10.0"))
3030
MIN_CUDA_VERSION = os.environ.get("RUNPOD_MIN_CUDA_VERSION", "11.8")
3131
NETWORK_CHECK_TIMEOUT = int(os.environ.get("RUNPOD_NETWORK_CHECK_TIMEOUT", "5"))
3232
GPU_BENCHMARK_TIMEOUT = int(os.environ.get("RUNPOD_GPU_BENCHMARK_TIMEOUT", "2"))
@@ -123,6 +123,8 @@ def _check_disk_space() -> None:
123123
"""
124124
Check disk space availability on root and /tmp.
125125
126+
Requires free space to be at least MIN_DISK_PERCENT% of total disk size.
127+
126128
Raises:
127129
RuntimeError: If insufficient disk space
128130
"""
@@ -134,16 +136,18 @@ def _check_disk_space() -> None:
134136
total_gb = usage.total / (1024**3)
135137
free_gb = usage.free / (1024**3)
136138
used_percent = 100 * (1 - free_gb / total_gb)
139+
free_percent = 100 * (free_gb / total_gb)
137140

138-
if free_gb < MIN_DISK_GB:
141+
# Check if free space is below the required percentage
142+
if free_percent < MIN_DISK_PERCENT:
139143
raise RuntimeError(
140-
f"Insufficient disk space on {path}: {free_gb:.2f}GB free, "
141-
f"{MIN_DISK_GB}GB required"
144+
f"Insufficient disk space on {path}: {free_gb:.2f}GB free "
145+
f"({free_percent:.1f}%), {MIN_DISK_PERCENT}% required"
142146
)
143147

144-
log.debug(
148+
log.info(
145149
f"Disk space check passed on {path}: {free_gb:.2f}GB free "
146-
f"({used_percent:.1f}% used)"
150+
f"({free_percent:.1f}% available)"
147151
)
148152
except FileNotFoundError:
149153
# /tmp may not exist on some systems

tests/test_serverless/test_modules/test_system_fitness.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,35 +97,38 @@ def test_memory_info_works(self):
9797
class TestDiskSpaceCheck:
9898
"""Tests for disk space checking."""
9999

100-
@patch("runpod.serverless.modules.rp_system_fitness.MIN_DISK_GB", 10.0)
100+
@patch("runpod.serverless.modules.rp_system_fitness.MIN_DISK_PERCENT", 10.0)
101101
@patch("shutil.disk_usage")
102102
def test_sufficient_disk_passes(self, mock_disk_usage):
103103
"""Test that sufficient disk space passes the check."""
104104
mock_usage = MagicMock()
105+
# 100GB total, 50GB free (50% free) - should pass with 10% minimum
105106
mock_usage.total = 100 * 1024**3
106107
mock_usage.free = 50 * 1024**3
107108
mock_disk_usage.return_value = mock_usage
108109

109110
# Should not raise
110111
_check_disk_space()
111112

112-
@patch("runpod.serverless.modules.rp_system_fitness.MIN_DISK_GB", 10.0)
113+
@patch("runpod.serverless.modules.rp_system_fitness.MIN_DISK_PERCENT", 10.0)
113114
@patch("shutil.disk_usage")
114115
def test_insufficient_disk_fails(self, mock_disk_usage):
115116
"""Test that insufficient disk space fails the check."""
116117
mock_usage = MagicMock()
117-
mock_usage.total = 20 * 1024**3
118+
# 100GB total, 5GB free (5% free) - should fail with 10% minimum
119+
mock_usage.total = 100 * 1024**3
118120
mock_usage.free = 5 * 1024**3
119121
mock_disk_usage.return_value = mock_usage
120122

121123
with pytest.raises(RuntimeError, match="Insufficient disk space"):
122124
_check_disk_space()
123125

124-
@patch("runpod.serverless.modules.rp_system_fitness.MIN_DISK_GB", 10.0)
126+
@patch("runpod.serverless.modules.rp_system_fitness.MIN_DISK_PERCENT", 10.0)
125127
@patch("shutil.disk_usage")
126128
def test_checks_both_root_and_tmp(self, mock_disk_usage):
127129
"""Test that both root and /tmp are checked."""
128130
mock_usage = MagicMock()
131+
# 100GB total, 50GB free (50% free) - should pass
129132
mock_usage.total = 100 * 1024**3
130133
mock_usage.free = 50 * 1024**3
131134
mock_disk_usage.return_value = mock_usage

0 commit comments

Comments
 (0)