Merge pull request #807 from Dstack-TEE/codex/nerdctl-compose-runner

feat: add nerdctl Compose runner with stargz support

Author: kvinwang

docs/nerdctl-compose.md

@@ -0,0 +1,63 @@
+# nerdctl Compose and lazy image pulling
+
+dstack supports two independent Compose runners. The runner is part of
+`app-compose.json`, so it is included in the compose hash and attestation.
+
+| `runner` | Image manager | `snapshotter` support |
+|---|---|---|
+| `docker-compose` | Docker Engine | none (Docker's overlayfs store) |
+| `nerdctl-compose` | containerd | `overlayfs` or `stargz` |
+
+## Deploy with stargz
+
+Build and push an eStargz image before deployment. For example, with Buildx:
+
+```bash
+docker buildx build -t registry.example.com/example/app:estargz \
+  --output type=registry,oci-mediatypes=true,compression=estargz,force-compression=true \
+  .
+```
+
+Reference that image from `docker-compose.yaml`, then deploy it with:
+
+```bash
+dstack deploy -c docker-compose.yaml \
+  --runner nerdctl-compose \
+  --snapshotter stargz
+```
+
+The resulting application manifest contains:
+
+```json
+{
+  "manifest_version": "3",
+  "runner": "nerdctl-compose",
+  "snapshotter": "stargz"
+}
+```
+
+Use containerd without lazy pulling by selecting overlayfs:
+
+```bash
+dstack deploy -c docker-compose.yaml \
+  --runner nerdctl-compose \
+  --snapshotter overlayfs
+```
+
+If `snapshotter` is omitted for `nerdctl-compose`, it defaults to `overlayfs`.
+Setting `snapshotter` with `docker-compose` is rejected rather than silently
+ignored.
+
+## Compatibility
+
+`nerdctl compose` implements the commonly used Docker Compose features, but it
+is not a drop-in implementation of every Docker-specific extension. Test
+applications that use Docker socket mounts, custom runtimes, or advanced
+networking before switching runners. The `nerdctl-compose` runner requires
+pre-built images and rejects Compose `build` sections. This avoids depending
+on an in-guest BuildKit daemon and ensures lazy-pull images were converted
+before deployment.
+
+Both backends keep their own image and container metadata. Changing the runner
+recreates the application through the selected backend; it does not migrate
+existing Docker containers into containerd.

dstack/crates/dstack-cli-core/src/compose.rs

@@ -13,10 +13,28 @@ use serde_json::json;
 /// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys);
 /// gateway and local-key-provider are off for the direct-port single-node flow.
 pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: bool) -> String {
-    let manifest = json!({
-        "manifest_version": 2,
+    build_app_compose_with_runtime(
+        name,
+        docker_compose_yaml,
+        kms_enabled,
+        "docker-compose",
+        None,
+    )
+}
+
+/// Build an app-compose manifest with an explicitly selected compose frontend.
+/// `snapshotter` is meaningful only for `nerdctl-compose`.
+pub fn build_app_compose_with_runtime(
+    name: &str,
+    docker_compose_yaml: &str,
+    kms_enabled: bool,
+    runner: &str,
+    snapshotter: Option<&str>,
+) -> String {
+    let mut manifest = json!({
+        "manifest_version": if runner == "nerdctl-compose" { json!("3") } else { json!(2) },
         "name": name,
-        "runner": "docker-compose",
+        "runner": runner,
         "docker_compose_file": docker_compose_yaml,
         "kms_enabled": kms_enabled,
         "gateway_enabled": false,
@@ -31,7 +49,30 @@ pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: boo
         // (NTS is also currently broken in guest images — see dstack#745.)
         "secure_time": false,
     });
+    if let Some(snapshotter) = snapshotter {
+        manifest["snapshotter"] = json!(snapshotter);
+    }
     // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical
     // to serde_json::to_string_pretty (avoids an expect on an unfailable Result).
     format!("{manifest:#}")
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn nerdctl_manifest_uses_v3_and_records_snapshotter() {
+        let body = build_app_compose_with_runtime(
+            "test",
+            "services: {}",
+            false,
+            "nerdctl-compose",
+            Some("stargz"),
+        );
+        let value: serde_json::Value = serde_json::from_str(&body).unwrap();
+        assert_eq!(value["manifest_version"], "3");
+        assert_eq!(value["runner"], "nerdctl-compose");
+        assert_eq!(value["snapshotter"], "stargz");
+    }
+}

dstack/crates/dstack-cli/src/main.rs

@@ -11,7 +11,7 @@
 //! `logs`, a global `-j/--json`).
 
 use anyhow::{bail, Context, Result};
-use clap::{Parser, Subcommand};
+use clap::{Parser, Subcommand, ValueEnum};
 use dstack_cli_core::layout::InstallLayout;
 use dstack_cli_core::vmm::{Vmm, DEFAULT_HOST};
 use dstack_cli_core::{compose, ports, rpc};
@@ -46,6 +46,37 @@ struct Cli {
     command: Command,
 }
 
+#[derive(Clone, Copy, Debug, Default, ValueEnum)]
+enum ComposeRunner {
+    #[default]
+    DockerCompose,
+    NerdctlCompose,
+}
+
+impl ComposeRunner {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::DockerCompose => "docker-compose",
+            Self::NerdctlCompose => "nerdctl-compose",
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, ValueEnum)]
+enum Snapshotter {
+    Overlayfs,
+    Stargz,
+}
+
+impl Snapshotter {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Overlayfs => "overlayfs",
+            Self::Stargz => "stargz",
+        }
+    }
+}
+
 #[derive(Subcommand)]
 enum Command {
     /// Deploy an app from a docker-compose file.
@@ -84,6 +115,12 @@ enum Command {
         /// build + hash the compose and print it, without deploying.
         #[arg(long)]
         dry_run: bool,
+        /// compose frontend used inside the guest.
+        #[arg(long, value_enum, default_value = "docker-compose")]
+        runner: ComposeRunner,
+        /// containerd snapshotter (supported only with --runner nerdctl-compose).
+        #[arg(long, value_enum)]
+        snapshotter: Option<Snapshotter>,
     },
     /// List deployed apps.
     Apps,
@@ -144,6 +181,8 @@ async fn main() -> Result<()> {
             no_kms,
             allowlist,
             dry_run,
+            runner,
+            snapshotter,
         } => {
             let compose = resolve_compose_arg(compose, compose_file)?;
             let image = if use_local_defaults {
@@ -174,6 +213,8 @@ async fn main() -> Result<()> {
                 allowlist.as_deref(),
                 dry_run,
                 json,
+                runner,
+                snapshotter,
             )
             .await
         }
@@ -347,10 +388,21 @@ async fn cmd_deploy(
     allowlist: Option<&str>,
     dry_run: bool,
     json: bool,
+    runner: ComposeRunner,
+    snapshotter: Option<Snapshotter>,
 ) -> Result<()> {
+    if matches!(runner, ComposeRunner::DockerCompose) && snapshotter.is_some() {
+        bail!("--snapshotter is only supported with --runner nerdctl-compose");
+    }
     let yaml = std::fs::read_to_string(compose_path)
         .with_context(|| format!("reading compose file '{compose_path}'"))?;
-    let app_compose = compose::build_app_compose(name, &yaml, !no_kms);
+    let app_compose = compose::build_app_compose_with_runtime(
+        name,
+        &yaml,
+        !no_kms,
+        runner.as_str(),
+        snapshotter.map(Snapshotter::as_str),
+    );
 
     let mut port_maps = Vec::new();
     for spec in port_specs {

dstack/dstack-types/src/lib.rs

@@ -80,6 +80,10 @@ pub struct AppCompose {
     #[serde(default)]
     pub features: Vec<String>,
     pub runner: String,
+    /// containerd snapshotter used by the `nerdctl-compose` runner.
+    /// The field is invalid for other runners.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub snapshotter: Option<ContainerSnapshotter>,
     #[serde(default)]
     pub docker_compose_file: Option<String>,
     #[serde(default)]
@@ -121,6 +125,13 @@ pub struct AppCompose {
     pub requirements: Option<Requirements>,
 }
 
+#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)]
+#[serde(rename_all = "kebab-case")]
+pub enum ContainerSnapshotter {
+    Overlayfs,
+    Stargz,
+}
+
 /// Canonical source for the policy used when `requirements.gpu_policy` is
 /// absent. Both typed defaults and measurement are derived from this JSON.
 pub const DEFAULT_GPU_POLICY: &str = "{}";
@@ -445,6 +456,26 @@ mod app_compose_tests {
         assert_eq!(compose.manifest_version_u32(), Some(3));
     }
 
+    #[test]
+    fn parses_supported_container_snapshotters() {
+        let compose: AppCompose = serde_json::from_value(serde_json::json!({
+            "manifest_version": "3",
+            "name": "test",
+            "runner": "nerdctl-compose",
+            "snapshotter": "stargz"
+        }))
+        .unwrap();
+        assert_eq!(compose.snapshotter, Some(ContainerSnapshotter::Stargz));
+
+        let invalid = serde_json::from_value::<AppCompose>(serde_json::json!({
+            "manifest_version": "3",
+            "name": "test",
+            "runner": "nerdctl-compose",
+            "snapshotter": "unknown"
+        }));
+        assert!(invalid.is_err());
+    }
+
     #[test]
     fn manifest_version_accepts_legacy_numeric_1_and_2() {
         assert_eq!(

dstack/dstack-util/src/system_setup.rs

@@ -839,6 +839,14 @@ fn verify_manifest_feature_requirements(app_compose: &AppCompose) -> Result<()>
             "requirements requires manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed"
         );
     }
+    if app_compose.runner == "nerdctl-compose" && manifest_version < MANIFEST_VERSION_3 {
+        bail!(
+            "nerdctl-compose requires manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed"
+        );
+    }
+    if app_compose.runner != "nerdctl-compose" && app_compose.snapshotter.is_some() {
+        bail!("snapshotter is only supported by the nerdctl-compose runner");
+    }
     Ok(())
 }
 
@@ -3162,6 +3170,27 @@ fn test_os_version_requirement_requires_v3_manifest() {
     assert!(err.to_string().contains("requires manifest_version"));
 }
 
+#[test]
+fn test_nerdctl_compose_requires_v3_manifest() {
+    let mut app_compose = test_app_compose(serde_json::json!(2), None, None);
+    app_compose.runner = "nerdctl-compose".to_string();
+    let err = verify_manifest_feature_requirements(&app_compose).unwrap_err();
+    assert!(err.to_string().contains("nerdctl-compose requires"));
+
+    app_compose.manifest_version = "3".to_string();
+    verify_manifest_feature_requirements(&app_compose).unwrap();
+}
+
+#[test]
+fn test_snapshotter_is_rejected_for_other_runners() {
+    let mut app_compose = test_app_compose(serde_json::json!("3"), None, None);
+    app_compose.snapshotter = Some(dstack_types::ContainerSnapshotter::Stargz);
+    let err = verify_manifest_feature_requirements(&app_compose).unwrap_err();
+    assert!(err
+        .to_string()
+        .contains("snapshotter is only supported by the nerdctl-compose runner"));
+}
+
 #[test]
 fn test_os_version_requirement_rejects_too_old_os() {
     let app_compose = test_app_compose(serde_json::json!("3"), Some(">=0.6.1"), None);

dstack/guest-agent/src/rpc_service.rs

@@ -735,6 +735,7 @@ mod tests {
             name: String::new(),
             features: Vec::new(),
             runner: String::new(),
+            snapshotter: None,
             docker_compose_file: None,
             public_logs: false,
             public_sysinfo: false,

os/common/rootfs/app-compose.service

@@ -1,15 +1,15 @@
 [Unit]
 Description=App Compose Service
-Wants=docker.service
-After=docker.service dstack-prepare.service dstack-guest-agent.service
+Wants=docker.service containerd.service containerd-stargz-grpc.service
+After=docker.service containerd.service containerd-stargz-grpc.service dstack-prepare.service dstack-guest-agent.service
 
 [Service]
 Type=oneshot
 RemainAfterExit=true
 EnvironmentFile=-/dstack/.host-shared/.decrypted-env
 WorkingDirectory=/dstack
 ExecStart=/bin/app-compose.sh
-ExecStop=/bin/docker compose stop
+ExecStop=/bin/app-compose.sh stop
 StandardOutput=journal+console
 StandardError=journal+console