refactor(volume): configure attachments in app compose

Author: kvinwang

docs/verity-volumes.md

@@ -47,6 +47,7 @@ This adds the following measured entry to `app-compose.json`:
 {
   "verity_volumes": [
     {
+      "source": "models.img",
       "verity_root": "a1b2c3d4...",
       "target": "/run/models"
     }
@@ -59,7 +60,7 @@ The target must be an absolute path on a writable guest filesystem, such as
 
 ## Guest activation
 
-Before the application starts, `dstack-volume`:
+Before the application starts, `dstack-volume mount-all app-compose.json`:
 
 1. Scans `/sys/class/block` for disks whose first partition, or whole disk when
    unpartitioned, starts with the `DSTACK_VOLUME` magic.
@@ -72,6 +73,11 @@ A required volume that is missing, malformed, or fails verification stops guest
 preparation. dm-verity continues verifying blocks lazily as the application
 reads them.
 
+For diagnostics, `dstack-volume scan` lists recognized disks and
+`dstack-volume status app-compose.json` compares requested roots with attached
+and active devices. A single entry can be activated with
+`dstack-volume mount app-compose.json INDEX`.
+
 ## Trust and limitations
 
 The root hash authenticates bytes, not availability. A malicious host can omit a

dstack/crates/dstack-cli-core/src/compose.rs

@@ -13,14 +13,14 @@ use serde_json::json;
 /// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys);
 /// gateway and local-key-provider are off for the direct-port single-node flow.
 ///
-/// `verity_volumes` is a list of `(verity_root, target)` pairs. Each becomes a
+/// `verity_volumes` is a list of `(source, verity_root, target)` tuples. Each becomes a
 /// measured `verity_volumes` entry, so the CVM only seeds content matching the
 /// attested root. Empty for a normal deploy.
 pub fn build_app_compose(
     name: &str,
     docker_compose_yaml: &str,
     kms_enabled: bool,
-    verity_volumes: &[(String, String)],
+    verity_volumes: &[(String, String, String)],
 ) -> String {
     let mut manifest = json!({
         "manifest_version": 2,
@@ -43,7 +43,7 @@ pub fn build_app_compose(
     if !verity_volumes.is_empty() {
         manifest["verity_volumes"] = json!(verity_volumes
             .iter()
-            .map(|(root, target)| json!({ "verity_root": root, "target": target }))
+            .map(|(source, root, target)| json!({ "source": source, "verity_root": root, "target": target }))
             .collect::<Vec<_>>());
     }
     // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical

dstack/crates/dstack-cli/src/main.rs

@@ -290,9 +290,9 @@ async fn cmd_verity(
     Ok(())
 }
 
-/// A parsed `--volume` spec: the disk to attach plus its measured verity entry.
+/// A parsed `--volume` spec. All fields become part of measured app-compose.
 struct VolumeSpec {
-    volume: rpc::VmVolume,
+    source: String,
     verity_root: String,
     target: String,
 }
@@ -327,12 +327,7 @@ fn parse_volume(spec: &str) -> Result<VolumeSpec> {
         bail!("target '{target}' must be an absolute path");
     }
     Ok(VolumeSpec {
-        // verity volumes are read-only by construction; a writable one would let a
-        // guest corrupt the shared backing file (the vmm also forces this).
-        volume: rpc::VmVolume {
-            source: name.to_string(),
-            read_only: true,
-        },
+        source: name.to_string(),
         verity_root: root.to_string(),
         target: target.to_string(),
     })
@@ -430,12 +425,11 @@ async fn cmd_deploy(
 
     // each --volume declares a measured verity_volumes entry, so the built
     // app-compose (and thus app_id) binds the attested roots.
-    let verity_volumes: Vec<(String, String)> = parsed_volumes
+    let verity_volumes: Vec<(String, String, String)> = parsed_volumes
         .iter()
-        .map(|v| (v.verity_root.clone(), v.target.clone()))
+        .map(|v| (v.source.clone(), v.verity_root.clone(), v.target.clone()))
         .collect();
     let app_compose = compose::build_app_compose(name, &yaml, !no_kms, &verity_volumes);
-    let volumes: Vec<_> = parsed_volumes.into_iter().map(|v| v.volume).collect();
 
     let mut cfg = rpc::VmConfiguration {
         name: name.to_string(),
@@ -445,7 +439,6 @@ async fn cmd_deploy(
         memory,
         disk_size: disk,
         ports: port_maps.clone(),
-        volumes,
         ..Default::default()
     };
 
@@ -675,8 +668,7 @@ mod tests {
     fn parses_volume_specs() {
         let root = "a".repeat(64);
         let data = parse_volume(&format!("weights.img:{root}:/models/llama")).unwrap();
-        assert_eq!(data.volume.source, "weights.img");
-        assert!(data.volume.read_only);
+        assert_eq!(data.source, "weights.img");
         assert_eq!(data.verity_root, root);
         assert_eq!(data.target, "/models/llama");
 

dstack/crates/dstack-volume/src/bin/dstack-volume.rs

@@ -41,22 +41,58 @@ struct VerityVolume {
 
 fn main() -> Result<()> {
     tracing_subscriber::fmt().init();
-    let compose_path = std::env::args_os()
-        .nth(1)
-        .map(PathBuf::from)
-        .unwrap_or_else(|| PathBuf::from("app-compose.json"));
+    let mut args = std::env::args_os().skip(1);
+    let command = args
+        .next()
+        .context(
+            "usage: dstack-volume <mount-all [COMPOSE] | mount COMPOSE INDEX | scan | status [COMPOSE]>",
+        )?;
+    match command.to_str() {
+        Some("mount-all") => mount_all(
+            args.next()
+                .map(PathBuf::from)
+                .unwrap_or_else(|| PathBuf::from("app-compose.json")),
+        ),
+        Some("mount") => {
+            let compose = args.next().context("mount requires COMPOSE and INDEX")?;
+            let index = args
+                .next()
+                .context("mount requires COMPOSE and INDEX")?
+                .to_str()
+                .context("INDEX is not UTF-8")?
+                .parse()
+                .context("invalid volume INDEX")?;
+            mount_one(PathBuf::from(compose), index)
+        }
+        Some("scan") => scan(),
+        Some("status") => status(
+            args.next()
+                .map(PathBuf::from)
+                .unwrap_or_else(|| PathBuf::from("app-compose.json")),
+        ),
+        _ => bail!("unknown command {:?}", command),
+    }
+}
+
+fn read_compose(compose_path: &Path) -> Result<AppCompose> {
     // Deserialize the complete shared type before probing any untrusted disk.
     // This validates roots and targets as part of parsing the measured compose.
-    let compose: AppCompose = serde_json::from_slice(&fs::read(&compose_path)?)
-        .with_context(|| format!("parsing {}", compose_path.display()))?;
-    if compose.verity_volumes.is_empty() {
-        return Ok(());
-    }
+    serde_json::from_slice(&fs::read(compose_path)?)
+        .with_context(|| format!("parsing {}", compose_path.display()))
+}
 
+fn prepare_volumes() -> Result<Vec<VerityVolume>> {
     let _ = run_cmd!(modprobe dm-verity);
     let _ = run_cmd!(udevadm settle --timeout=5);
+    discover_volumes()
+}
 
-    let volumes = discover_volumes()?;
+fn mount_all(compose_path: PathBuf) -> Result<()> {
+    let compose = read_compose(&compose_path)?;
+    if compose.verity_volumes.is_empty() {
+        return Ok(());
+    }
+    let volumes = prepare_volumes()?;
     info!(
         found = volumes.len(),
         requested = compose.verity_volumes.len(),
@@ -74,6 +110,53 @@ fn main() -> Result<()> {
     Ok(())
 }
 
+fn mount_one(compose_path: PathBuf, index: usize) -> Result<()> {
+    let compose = read_compose(&compose_path)?;
+    let requested = compose
+        .verity_volumes
+        .get(index)
+        .with_context(|| format!("volume index {index} is out of range"))?;
+    let volumes = prepare_volumes()?;
+    activate_requested(index, requested, &volumes, &mut HashSet::new()).with_context(|| {
+        format!(
+            "failed to activate required volume {index} at {}",
+            requested.target.display()
+        )
+    })
+}
+
+fn scan() -> Result<()> {
+    for volume in prepare_volumes()? {
+        println!(
+            "{}\tdata={}\thash={}",
+            hex::encode(volume.root_hash),
+            volume.data.display(),
+            volume.hash.display()
+        );
+    }
+    Ok(())
+}
+
+fn status(compose_path: PathBuf) -> Result<()> {
+    let compose = read_compose(&compose_path)?;
+    let volumes = prepare_volumes()?;
+    for (index, requested) in compose.verity_volumes.iter().enumerate() {
+        let attached = volumes
+            .iter()
+            .any(|volume| volume.root_hash == requested.verity_root);
+        let mapper_name = format!("dstack-verity{index}");
+        let active = Path::new("/dev/mapper").join(&mapper_name).exists()
+            && mapping_root(&mapper_name)?
+                .eq_ignore_ascii_case(&hex::encode(requested.verity_root));
+        println!(
+            "{index}\troot={}\ttarget={}\tattached={attached}\tactive={active}",
+            hex::encode(requested.verity_root),
+            requested.target.display()
+        );
+    }
+    Ok(())
+}
+
 fn discover_volumes() -> Result<Vec<VerityVolume>> {
     let mut found = Vec::new();
     let disks = list_disks()?;

dstack/crates/dstack-volume/src/volume_format.rs

@@ -65,6 +65,7 @@ mod tests {
     fn verity_volume_validates_root_and_target_during_deserialization(
     ) -> Result<(), serde_json::Error> {
         let volume: VerityVolume = serde_json::from_value(serde_json::json!({
+            "source": "models.img",
             "verity_root": "5a".repeat(32),
             "target": "/run/models"
         }))?;
@@ -76,11 +77,13 @@ mod tests {
         );
 
         assert!(serde_json::from_value::<VerityVolume>(serde_json::json!({
+            "source": "models.img",
             "verity_root": "abcd",
             "target": "/run/models"
         }))
         .is_err());
         assert!(serde_json::from_value::<VerityVolume>(serde_json::json!({
+            "source": "models.img",
             "verity_root": "5a".repeat(32),
             "target": "relative/path"
         }))

dstack/dstack-types/src/lib.rs

@@ -130,6 +130,8 @@ pub struct AppCompose {
 /// A pre-baked, read-only dm-verity volume attached to the CVM.
 #[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct VerityVolume {
+    /// Bare image file name resolved by the VMM under `cvm.volumes_dir`.
+    pub source: String,
     /// dm-verity root hash (hex): the volume's content identity and integrity
     /// check. The guest matches attached devices against it.
     #[serde(with = "hex_bytes")]

dstack/vmm/rpc/proto/vmm_rpc.proto

@@ -118,17 +118,7 @@ message VmConfiguration {
   optional NetworkingConfig networking = 18;
   // Per-VM networking overrides. If set, wins over singular networking.
   repeated NetworkingConfig networks = 19;
-  // Extra volumes to attach (e.g. pre-baked verity volumes). The host only
-  // supplies bytes; the guest verifies content against the measured verity_root.
-  repeated VmVolume volumes = 20;
-}
-
-// An extra disk attached to the VM, such as a pre-baked verity volume.
-message VmVolume {
-  // Volume file name, resolved against the vmm's configured cvm.volumes_dir.
-  string source = 1;
-  // Attach the disk read-only.
-  bool read_only = 2;
+  reserved 20;
 }
 
 // Per-VM networking configuration.

dstack/vmm/src/app/vm_info.rs

@@ -169,15 +169,6 @@ impl VmInfo {
                     no_tee,
                     networking: configured_networking,
                     networks: configured_networks,
-                    volumes: self
-                        .manifest
-                        .volumes
-                        .iter()
-                        .map(|volume| pb::VmVolume {
-                            source: volume.source.clone(),
-                            read_only: volume.read_only,
-                        })
-                        .collect(),
                 })
             },
             app_url: self

dstack/vmm/src/main_service.rs

@@ -184,7 +184,9 @@ pub fn create_manifest_from_vm_config(
         Some(gpus) => resolve_gpus_with_config(gpus, cvm_config)?,
         None => GpuConfig::default(),
     };
-    let volumes = resolve_volumes(&request.volumes, cvm_config)?;
+    let app_compose: AppCompose = serde_json::from_str(&request.compose_file)
+        .context("invalid app-compose in VM configuration")?;
+    let volumes = resolve_volumes(&app_compose.verity_volumes, cvm_config)?;
 
     Ok(Manifest {
         id,
@@ -211,7 +213,7 @@ pub fn create_manifest_from_vm_config(
 /// bare file name under that directory; the host attaches the bytes, and the
 /// guest verifies content against the measured `verity_root`.
 fn resolve_volumes(
-    reqs: &[rpc::VmVolume],
+    reqs: &[dstack_types::VerityVolume],
     cvm_config: &crate::config::CvmConfig,
 ) -> Result<Vec<crate::app::VmVolume>> {
     if reqs.is_empty() {
@@ -903,7 +905,8 @@ mod tests {
         VmConfiguration {
             name: "vm-test".to_string(),
             image: "dstack-test".to_string(),
-            compose_file: "{}".to_string(),
+            compose_file: r#"{"manifest_version":2,"name":"test","runner":"docker-compose"}"#
+                .to_string(),
             vcpu: 1,
             memory: 1024,
             disk_size: 10,
@@ -920,7 +923,6 @@ mod tests {
             no_tee: false,
             networking: None,
             networks: vec![],
-            volumes: vec![],
         }
     }
 
@@ -1014,9 +1016,10 @@ mod tests {
         cvm_config.volumes_dir = tmp.path().to_string_lossy().into_owned();
 
         let volumes = resolve_volumes(
-            &[rpc::VmVolume {
+            &[dstack_types::VerityVolume {
                 source: "volume.img".into(),
-                read_only: false,
+                verity_root: [0; 32],
+                target: "/run/volume".into(),
             }],
             &cvm_config,
         )?;