Skip to content

Commit d186e06

Browse files
LeechaelLeechael
authored
Merge pull request #358 from Dstack-TEE/swap
cvm: Add built-in swap config
2 parents e3c0cc7 + 26d4c2d commit d186e06

7 files changed

Lines changed: 218 additions & 20 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dstack-types/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,4 @@ license.workspace = true
1313
serde = { workspace = true, features = ["derive"] }
1414
serde-human-bytes.workspace = true
1515
sha3.workspace = true
16+
size-parser = { workspace = true, features = ["serde"] }

‎dstack-types/src/lib.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
use serde::{Deserialize, Serialize};
66
use serde_human_bytes as hex_bytes;
7+
use size_parser::human_size;
78

89
#[derive(Deserialize, Serialize, Debug, Clone)]
910
pub struct AppCompose {
@@ -41,6 +42,8 @@ pub struct AppCompose {
4142
pub secure_time: bool,
4243
#[serde(default)]
4344
pub storage_fs: Option<String>,
45+
#[serde(default, with = "human_size")]
46+
pub swap_size: u64,
4447
}
4548

4649
fn default_true() -> bool {

‎dstack-util/src/system_setup.rs‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use std::{
99
path::{Path, PathBuf},
1010
process::Command,
1111
str::FromStr,
12+
time::Duration,
1213
};
1314

1415
use anyhow::{anyhow, bail, Context, Result};
@@ -685,6 +686,85 @@ impl<'a> Stage0<'a> {
685686
}
686687
}
687688

689+
async fn setup_swap(&self, swap_size: u64, opts: &DstackOptions) -> Result<()> {
690+
match opts.storage_fs {
691+
FsType::Zfs => self.setup_swap_zvol(swap_size).await,
692+
FsType::Ext4 => self.setup_swapfile(swap_size).await,
693+
}
694+
}
695+
696+
async fn setup_swapfile(&self, swap_size: u64) -> Result<()> {
697+
let swapfile = self.args.mount_point.join("swapfile");
698+
if swapfile.exists() {
699+
fs::remove_file(&swapfile).context("Failed to remove swapfile")?;
700+
info!("Removed existing swapfile");
701+
}
702+
if swap_size == 0 {
703+
return Ok(());
704+
}
705+
let swapfile = swapfile.display().to_string();
706+
info!("Creating swapfile at {swapfile} (size {swap_size} bytes)");
707+
let size_str = swap_size.to_string();
708+
cmd! {
709+
fallocate -l $size_str $swapfile;
710+
chmod 600 $swapfile;
711+
mkswap $swapfile;
712+
swapon $swapfile;
713+
swapon --show;
714+
}
715+
.context("Failed to enable swap on swapfile")?;
716+
Ok(())
717+
}
718+
719+
async fn setup_swap_zvol(&self, swap_size: u64) -> Result<()> {
720+
let swapvol_path = "dstack/swap";
721+
let swapvol_device_path = format!("/dev/zvol/{swapvol_path}");
722+
723+
if Path::new(&swapvol_device_path).exists() {
724+
cmd! {
725+
zfs set volmode=none $swapvol_path;
726+
zfs destroy $swapvol_path;
727+
}
728+
.context("Failed to destroy swap zvol")?;
729+
}
730+
731+
if swap_size == 0 {
732+
return Ok(());
733+
}
734+
735+
info!("Creating swap zvol at {swapvol_device_path} (size {swap_size} bytes)");
736+
737+
let size_str = swap_size.to_string();
738+
cmd! {
739+
zfs create -V $size_str
740+
-o compression=zle
741+
-o logbias=throughput
742+
-o sync=always
743+
-o primarycache=metadata
744+
-o com.sun:auto-snapshot=false
745+
$swapvol_path
746+
}
747+
.with_context(|| format!("Failed to create swap zvol {swapvol_path}"))?;
748+
749+
let mut count = 0u32;
750+
while !Path::new(&swapvol_device_path).exists() && count < 10 {
751+
std::thread::sleep(Duration::from_secs(1));
752+
count += 1;
753+
}
754+
if !Path::new(&swapvol_device_path).exists() {
755+
bail!("Device {swapvol_device_path} did not appear after 10 seconds");
756+
}
757+
758+
cmd! {
759+
mkswap $swapvol_device_path;
760+
swapon $swapvol_device_path;
761+
swapon --show;
762+
}
763+
.context("Failed to enable swap on zvol")?;
764+
765+
Ok(())
766+
}
767+
688768
async fn mount_data_disk(
689769
&self,
690770
initialized: bool,
@@ -958,13 +1038,14 @@ impl<'a> Stage0<'a> {
9581038
opts.storage_encrypted, opts.storage_fs
9591039
);
9601040

961-
self.vmm.notify_q("boot.progress", "unsealing env").await;
9621041
self.mount_data_disk(
9631042
is_initialized,
9641043
&hex::encode(&app_keys.disk_crypt_key),
9651044
&opts,
9661045
)
9671046
.await?;
1047+
self.setup_swap(self.shared.app_compose.swap_size, &opts)
1048+
.await?;
9681049
self.vmm
9691050
.notify_q(
9701051
"instance.info",

‎vmm/src/console.html‎

Lines changed: 99 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -711,6 +711,19 @@ <h2>Deploy a new instance</h2>
711711
</div>
712712
</div>
713713

714+
<div class="form-group">
715+
<label for="swapSize">Swap (optional)</label>
716+
<div style="display: flex; align-items: center; gap: 8px;">
717+
<input id="swapSize" v-model.number="vmForm.swapValue" type="number" min="0"
718+
step="0.1" placeholder="Swap size" style="flex: 1;">
719+
<select v-model="vmForm.swapUnit" style="width: 80px;">
720+
<option value="MB">MB</option>
721+
<option value="GB">GB</option>
722+
</select>
723+
</div>
724+
<small style="color: #666;">Leave as 0 to disable swap.</small>
725+
</div>
726+
714727
<div class="form-group">
715728
<label for="diskSize">Storage (GB)</label>
716729
<input id="diskSize" v-model.number="vmForm.disk_size" type="number"
@@ -1009,6 +1022,10 @@ <h4 style="margin: 0 0 12px 0;">VM Configuration</h4>
10091022
<span class="detail-value">{{ formatMemory(vm.configuration?.memory)
10101023
}}</span>
10111024
</div>
1025+
<div class="detail-item" v-if="vm.configuration?.swap_size">
1026+
<span class="detail-label">Swap:</span>
1027+
<span class="detail-value">{{ formatMemory(bytesToMB(vm.configuration.swap_size)) }}</span>
1028+
</div>
10121029
<div class="detail-item">
10131030
<span class="detail-label">Disk Size:</span>
10141031
<span class="detail-value">{{ vm.configuration?.disk_size }} GB</span>
@@ -1136,6 +1153,21 @@ <h3>Update VM Config</h3>
11361153
</div>
11371154
</div>
11381155

1156+
<div class="form-group">
1157+
<label for="upgradeSwap">Swap (optional)</label>
1158+
<div style="display: flex; align-items: center; gap: 8px;">
1159+
<input id="upgradeSwap" v-model.number="upgradeDialog.swapValue" type="number" min="0"
1160+
step="0.1" placeholder="Swap size" style="flex: 1;"
1161+
:disabled="!upgradeDialog.updateCompose">
1162+
<select v-model="upgradeDialog.swapUnit" style="width: 80px;"
1163+
:disabled="!upgradeDialog.updateCompose">
1164+
<option value="MB">MB</option>
1165+
<option value="GB">GB</option>
1166+
</select>
1167+
</div>
1168+
<small style="color: #666;">Enable "Update compose" to change swap size.</small>
1169+
</div>
1170+
11391171
<div class="form-group">
11401172
<label for="diskSize">Disk Size (GB)</label>
11411173
<input id="diskSize" v-model.number="upgradeDialog.disk_size" type="number"
@@ -1527,6 +1559,9 @@ <h3>Derive VM</h3>
15271559
memory: 2048, // This will be computed from memoryValue and memoryUnit
15281560
memoryValue: 2,
15291561
memoryUnit: 'GB',
1562+
swap_size: 0,
1563+
swapValue: 0,
1564+
swapUnit: 'GB',
15301565
disk_size: 20,
15311566
selectedGpus: [],
15321567
attachAllGpus: false,
@@ -1566,6 +1601,9 @@ <h3>Derive VM</h3>
15661601
memory: 0, // This will be computed from memoryValue and memoryUnit
15671602
memoryValue: 0,
15681603
memoryUnit: 'MB',
1604+
swap_size: 0,
1605+
swapValue: 0,
1606+
swapUnit: 'GB',
15691607
disk_size: 0,
15701608
image: '',
15711609
ports: [],
@@ -1724,6 +1762,11 @@ <h3>Derive VM</h3>
17241762
app_compose.pre_launch_script = vmForm.value.preLaunchScript;
17251763
}
17261764

1765+
const swapBytes = Math.max(0, Math.round(vmForm.value.swap_size || 0));
1766+
if (swapBytes > 0) {
1767+
app_compose.swap_size = swapBytes;
1768+
}
1769+
17271770
// If APP_LAUNCH_TOKEN is set, add it's sha256 hash to the app compose file
17281771
const launchToken = vmForm.value.encryptedEnvs.find(env => env.key === 'APP_LAUNCH_TOKEN');
17291772
if (launchToken) {
@@ -1831,15 +1874,39 @@ <h3>Derive VM</h3>
18311874
showCreateDialog.value = true;
18321875
vmForm.value.encryptedEnvs = [];
18331876
vmForm.value.app_id = null;
1877+
vmForm.value.swapValue = 0;
1878+
vmForm.value.swapUnit = 'GB';
1879+
vmForm.value.swap_size = 0;
18341880
loadGpus();
18351881
};
18361882

18371883
// Memory conversion functions
18381884
const convertMemoryToMB = (value, unit) => {
1885+
const numericValue = Number(value);
1886+
if (!Number.isFinite(numericValue) || numericValue < 0) {
1887+
return 0;
1888+
}
18391889
if (unit === 'GB') {
1840-
return value * 1024;
1890+
return numericValue * 1024;
18411891
}
1842-
return value;
1892+
return numericValue;
1893+
};
1894+
1895+
const BYTES_PER_MB = 1024 * 1024;
1896+
1897+
const convertSwapToBytes = (value, unit) => {
1898+
const mb = convertMemoryToMB(value, unit);
1899+
if (!Number.isFinite(mb) || mb <= 0) {
1900+
return 0;
1901+
}
1902+
return Math.max(0, Math.round(mb * BYTES_PER_MB));
1903+
};
1904+
1905+
const bytesToMB = (bytes) => {
1906+
if (!bytes) {
1907+
return 0;
1908+
}
1909+
return bytes / BYTES_PER_MB;
18431910
};
18441911

18451912
const formatMemory = (memoryMB) => {
@@ -1862,6 +1929,14 @@ <h3>Derive VM</h3>
18621929
upgradeDialog.value.memory = convertMemoryToMB(upgradeDialog.value.memoryValue, upgradeDialog.value.memoryUnit);
18631930
});
18641931

1932+
watch([() => vmForm.value.swapValue, () => vmForm.value.swapUnit], () => {
1933+
vmForm.value.swap_size = convertSwapToBytes(vmForm.value.swapValue, vmForm.value.swapUnit);
1934+
});
1935+
1936+
watch([() => upgradeDialog.value.swapValue, () => upgradeDialog.value.swapUnit], () => {
1937+
upgradeDialog.value.swap_size = convertSwapToBytes(upgradeDialog.value.swapValue, upgradeDialog.value.swapUnit);
1938+
});
1939+
18651940
const createVm = async () => {
18661941
try {
18671942
// Convert memory based on selected unit
@@ -1879,6 +1954,12 @@ <h3>Derive VM</h3>
18791954
user_config: vmForm.value.user_config,
18801955
gpus: configGpu(vmForm.value),
18811956
};
1957+
const swapBytes = Math.max(0, Math.round(form.swap_size || 0));
1958+
if (swapBytes > 0) {
1959+
form.swap_size = swapBytes;
1960+
} else {
1961+
delete form.swap_size;
1962+
}
18821963
const _response = await rpcCall('CreateVm', form);
18831964
loadVMList();
18841965
showCreateDialog.value = false;
@@ -2054,6 +2135,10 @@ <h3>Derive VM</h3>
20542135
const selectedGpuSlots = currentGpuConfig.gpus?.map(gpu => gpu.slot) || [];
20552136
const appCompose = JSON.parse(updatedVM.configuration?.compose_file || "{}");
20562137

2138+
const swapBytes = Number(appCompose.swap_size || 0);
2139+
const swapMb = bytesToMB(swapBytes);
2140+
const swapDisplay = autoMemoryDisplay(swapMb);
2141+
20572142
upgradeDialog.value = {
20582143
show: true,
20592144
vm: updatedVM,
@@ -2066,6 +2151,9 @@ <h3>Derive VM</h3>
20662151
vcpu: updatedVM.configuration?.vcpu || 1,
20672152
memory: updatedVM.configuration?.memory || 1024,
20682153
...autoMemoryDisplay(updatedVM.configuration?.memory),
2154+
swap_size: swapBytes,
2155+
swapValue: Number(swapDisplay.memoryValue),
2156+
swapUnit: swapDisplay.memoryUnit,
20692157
disk_size: updatedVM.configuration?.disk_size || 10,
20702158
image: updatedVM.configuration?.image || '',
20712159
ports: updatedVM.configuration?.ports?.map(port => ({ ...port })) || [],
@@ -2100,6 +2188,13 @@ <h3>Derive VM</h3>
21002188
}
21012189
}
21022190
app_compose.pre_launch_script = upgradeDialog.value.preLaunchScript?.trim();
2191+
2192+
const swapBytes = Math.max(0, Math.round(upgradeDialog.value.swap_size || 0));
2193+
if (swapBytes > 0) {
2194+
app_compose.swap_size = swapBytes;
2195+
} else {
2196+
delete app_compose.swap_size;
2197+
}
21032198
return JSON.stringify(app_compose);
21042199
}
21052200

@@ -2493,6 +2588,7 @@ <h3>Derive VM</h3>
24932588
composeHashPreview,
24942589
upgradeComposeHashPreview,
24952590
formatMemory,
2591+
bytesToMB,
24962592
// Pagination and search variables
24972593
searchQuery,
24982594
currentPage,
@@ -2514,4 +2610,4 @@ <h3>Derive VM</h3>
25142610
</script>
25152611
</body>
25162612

2517-
</html>
2613+
</html>

‎vmm/src/main_service.rs‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -173,22 +173,22 @@ pub fn create_manifest_from_vm_config(
173173
None => GpuConfig::default(),
174174
};
175175

176-
Ok(Manifest::builder()
177-
.id(id)
178-
.name(request.name.clone())
179-
.app_id(app_id)
180-
.image(request.image.clone())
181-
.vcpu(request.vcpu)
182-
.memory(request.memory)
183-
.disk_size(request.disk_size)
184-
.port_map(port_map)
185-
.created_at_ms(now)
186-
.hugepages(request.hugepages)
187-
.pin_numa(request.pin_numa)
188-
.gpus(gpus)
189-
.kms_urls(request.kms_urls.clone())
190-
.gateway_urls(request.gateway_urls.clone())
191-
.build())
176+
Ok(Manifest {
177+
id,
178+
name: request.name.clone(),
179+
app_id,
180+
vcpu: request.vcpu,
181+
memory: request.memory,
182+
disk_size: request.disk_size,
183+
image: request.image.clone(),
184+
port_map,
185+
created_at_ms: now,
186+
hugepages: request.hugepages,
187+
pin_numa: request.pin_numa,
188+
gpus: Some(gpus),
189+
kms_urls: request.kms_urls.clone(),
190+
gateway_urls: request.gateway_urls.clone(),
191+
})
192192
}
193193

194194
impl RpcHandler {

0 commit comments

Comments
 (0)