[docker-in-docker] - Fixing issue with containerd (#1653)

* [docker-in-docker] - Fixing issue with containerd

* Change the base image used for stress test as its not compatible with ubuntu resolute

* Change test base image

Author: Kaniska244

.github/workflows/docker-in-docker-stress-test.yaml

@@ -19,7 +19,7 @@ jobs:
         run: npm install -g @devcontainers/cli
 
       - name: "Generating tests for 'docker-in-docker' which validates if docker daemon is running"
-        run: devcontainer features test --skip-scenarios -f docker-in-docker -i mcr.microsoft.com/devcontainers/base:ubuntu .
+        run: devcontainer features test --skip-scenarios -f docker-in-docker -i mcr.microsoft.com/devcontainers/base:noble .
 
   test-onCreate:
     strategy:
@@ -34,4 +34,4 @@ jobs:
         run: npm install -g @devcontainers/cli
 
       - name: "Generating tests for 'docker-in-docker' which validates if docker daemon is available within 'onCreateCommand'"
-        run: devcontainer features test -f docker-in-docker --skip-autogenerated --filter "docker_with_on_create_command"
+        run: devcontainer features test -f docker-in-docker --skip-autogenerated --filter "docker_with_on_create_command" -i mcr.microsoft.com/devcontainers/base:noble

.github/workflows/test-pr-arm64.yaml

@@ -47,7 +47,9 @@ jobs:
           ]
         exclude:
           - features: docker-in-docker
-            baseImage: mcr.microsoft.com/devcontainers/base:debian           
+            baseImage: mcr.microsoft.com/devcontainers/base:debian
+          - features: docker-in-docker
+            baseImage: mcr.microsoft.com/devcontainers/base:ubuntu       
     steps:
       - uses: actions/checkout@v6
 

.github/workflows/test-pr.yaml

@@ -65,7 +65,9 @@ jobs:
           - features: docker-in-docker
             baseImage: mcr.microsoft.com/devcontainers/base:debian         
           - features: docker-outside-of-docker
-            baseImage: mcr.microsoft.com/devcontainers/base:debian             
+            baseImage: mcr.microsoft.com/devcontainers/base:debian   
+          - features: docker-in-docker
+            baseImage: mcr.microsoft.com/devcontainers/base:ubuntu 
     steps:
       - uses: actions/checkout@v6
 

src/docker-in-docker/NOTES.md

@@ -18,3 +18,14 @@ Debian Trixie (13) does not include moby-cli and related system packages, so the
 Ubuntu 26.04 (Resolute) does not currently have moby packages available, so the feature cannot install with "moby": "true". To use this feature on Resolute, please set "moby": "false". Additionally, the kernel on Ubuntu 26.04 no longer supports legacy iptables NAT tables, so the feature automatically falls back to `iptables-nft` when `iptables-legacy` is not functional.
 
 `bash` is required to execute the `install.sh` script.
+
+## Persisted state
+
+This Feature mounts two named Docker volumes into the dev container so that the daemons have writable, non-overlay storage for their state:
+
+* `dind-var-lib-docker-${devcontainerId}` → `/var/lib/docker`
+* `dind-var-lib-containerd-${devcontainerId}` → `/var/lib/containerd`
+
+The `/var/lib/containerd` mount is required when the dev container's root filesystem is itself an overlayfs mount (the default in Kubernetes / containerd-backed hosts, GitHub Codespaces, and Docker with the containerd image store enabled). Without it, the standalone `containerd` started by this Feature would place its overlayfs snapshotter data on an overlay rootfs, causing overlay-on-overlay mounts to fail with `invalid argument`. See [issue #1639](https://github.com/devcontainers/features/issues/1639) for background.
+
+Because both volumes are scoped to `${devcontainerId}`, each dev container gets its own state and rebuilds preserve images and snapshots. Removing the dev container does not automatically remove these volumes; clean them up with `docker volume rm` if you want to reclaim space.

src/docker-in-docker/devcontainer-feature.json

@@ -1,6 +1,6 @@
 {
   "id": "docker-in-docker",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "name": "Docker (Docker-in-Docker)",
   "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker",
   "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.",
@@ -86,6 +86,11 @@
       "source": "dind-var-lib-docker-${devcontainerId}",
       "target": "/var/lib/docker",
       "type": "volume"
+    },
+    {
+      "source": "dind-var-lib-containerd-${devcontainerId}",
+      "target": "/var/lib/containerd",
+      "type": "volume"
     }
   ],
   "installsAfter": [

test/docker-in-docker/Dockerfile

@@ -1 +1 @@
-FROM ubuntu:focal
+FROM ubuntu:noble

test/docker-in-docker/overlayfs_containerd_root.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Regression test for devcontainers/features#1639 / PR #1645 follow-up:
+# verifies that when the dev container's root filesystem is overlayfs
+# (the default under Docker / containerd-backed hosts), the standalone
+# containerd started by the docker-in-docker Feature does NOT place its
+# overlayfs snapshotter data on an overlay rootfs (which would fail with
+# `invalid argument` when pulling images).
+#
+set -e
+
+source dev-container-features-test-lib
+
+# 1. Confirm we're really reproducing the affected condition:
+#    the dev container's / must be overlay.
+check "rootfs is overlay (precondition)" \
+    bash -c '[ "$(findmnt -no FSTYPE /)" = "overlay" ]'
+
+# 2. The Feature's volume mount must shadow /var/lib/containerd with a
+#    non-overlay filesystem. Without the mount, containerd's overlayfs
+#    snapshotter would be writing onto the overlay rootfs and fail at
+#    pull time.
+check "/var/lib/containerd is not overlay" \
+    bash -c '[ "$(findmnt -no FSTYPE /var/lib/containerd)" != "overlay" ]'
+
+check "/var/lib/docker is not overlay" \
+    bash -c '[ "$(findmnt -no FSTYPE /var/lib/docker)" != "overlay" ]'
+
+# 3. The actual symptom: pulling and running an image must succeed.
+#    Pre-PR-#1645 this fails with:
+#      failed to mount /tmp/containerd-mountXXXXX ... err: invalid argument
+check "docker run hello-world" \
+    docker run --rm hello-world
+
+# 4. Belt-and-braces: confirm dockerd is actually using the
+#    containerd-snapshotter path so we know this test exercises the
+#    affected code path, not the legacy overlay2 driver.
+check "containerd-snapshotter active" \
+    bash -c "docker info 2>/dev/null | grep -qiE 'driver-type: io.containerd.snapshotter.v1|Storage Driver: overlayfs'"
+
+reportResults
+

test/docker-in-docker/scenarios.json

@@ -1,4 +1,13 @@
 {
+    "overlayfs_containerd_root": {
+        "image": "mcr.microsoft.com/devcontainers/base:noble",
+        "features": {
+            "docker-in-docker": {
+                "version": "latest",
+                "moby": true
+            }
+        }
+    },     
     "docker_build_fallback_compose": {
         "image": "ubuntu:noble",
         "features": {
@@ -108,7 +117,7 @@
         }
     },
     "docker_python_bookworm": {
-        "image": "mcr.microsoft.com/devcontainers/base:bookworm",
+        "image": "mcr.microsoft.com/devcontainers/base:2.1.8-bookworm",
         "features": {
             "docker-in-docker": {
                 "moby": true,