Two devenv patches

.github/workflows/test-devcontainer.yml

@@ -0,0 +1,66 @@
+name: Test DevContainer
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    paths:
+      - 'devenv/**'
+      - 'common/.devcontainer/**'
+      - '.github/workflows/test-devcontainer.yml'
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [debian]
+        # TODO: c10s has PAM/sudo issues with devcontainer CLI's --userns=keep-id
+        # include:
+        #   - os: c10s
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up runner
+        uses: bootc-dev/actions/bootc-ubuntu-setup@main
+
+      - name: Build devcontainer image
+        run: just devenv-build-${{ matrix.os }}
+
+      - name: Create override config for local image
+        run: |
+          cat > /tmp/devcontainer-override.json << 'EOF'
+          {
+            "image": "localhost/bootc-devenv-${{ matrix.os }}:latest",
+            "runArgs": [
+              "--security-opt", "label=disable",
+              "--security-opt", "unmask=/proc/*",
+              "--device", "/dev/net/tun",
+              "--device", "/dev/kvm"
+            ],
+            "postCreateCommand": {
+              "devenv-init": "sudo /usr/local/bin/devenv-init.sh"
+            }
+          }
+          EOF
+
+      - name: Start devcontainer
+        run: |
+          npx --yes @devcontainers/cli up \
+            --workspace-folder . \
+            --docker-path podman \
+            --override-config /tmp/devcontainer-override.json \
+            --remove-existing-container
+
+      - name: Test nested podman in devcontainer
+        run: |
+          npx @devcontainers/cli exec \
+            --workspace-folder . \
+            --docker-path podman \
+            /usr/libexec/devenv-selftest.sh

Justfile

@@ -12,3 +12,26 @@ devenv-build-c10s:
 
 # Build devenv image with local tag (defaults to Debian)
 devenv-build: devenv-build-debian
+
+# Test devcontainer with a locally built image
+# Usage: just devcontainer-test <os>
+# Example: just devcontainer-test debian
+devcontainer-test os:
+	#!/bin/bash
+	set -euo pipefail
+	cat > /tmp/devcontainer-override.json << 'EOF'
+	{
+	  "image": "localhost/bootc-devenv-{{os}}:latest",
+	  "runArgs": [
+	    "--security-opt", "label=disable",
+	    "--security-opt", "unmask=/proc/*",
+	    "--device", "/dev/net/tun",
+	    "--device", "/dev/kvm"
+	  ],
+	  "postCreateCommand": {
+	    "devenv-init": "sudo /usr/local/bin/devenv-init.sh"
+	  }
+	}
+	EOF
+	npx --yes @devcontainers/cli up --workspace-folder . --docker-path podman --override-config /tmp/devcontainer-override.json --remove-existing-container
+	npx @devcontainers/cli exec --workspace-folder . --docker-path podman /usr/libexec/devenv-selftest.sh

common/.devcontainer/devcontainer.json

@@ -13,9 +13,14 @@
   },
   "features": {},
   "runArgs": [
-    // Because we want to be able to run podman and also use e.g. /dev/kvm
-    // among other things
-    "--privileged"
+    // Minimal security options for nested podman (avoids --privileged):
+    // - label=disable: Required for mounting /proc in nested user namespace
+    // - unmask=/proc/*: Allows access to /proc paths needed for nested containers
+    "--security-opt", "label=disable",
+    "--security-opt", "unmask=/proc/*",
+    // Device access for nested containers and VMs
+    "--device", "/dev/net/tun",
+    "--device", "/dev/kvm"
   ],
   "postCreateCommand": {
     // Our init script

common/.github/actions/bootc-ubuntu-setup/action.yml

@@ -14,7 +14,8 @@ runs:
       run: |
         set -xeuo pipefail
         sudo df -h
-        unwanted_pkgs=('^aspnetcore-.*' '^dotnet-.*' '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*'
+        # Use globs for package patterns (apt and dpkg both support fnmatch globs)
+        unwanted_pkgs=('aspnetcore-*' 'dotnet-*' 'llvm-*' 'php*' 'mongodb-*' 'mysql-*'
                        azure-cli google-chrome-stable firefox mono-devel)
         unwanted_dirs=(/usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL)
         # Start background removal operations as systemd units; if this causes
@@ -30,9 +31,12 @@ runs:
         for x in ${unwanted_dirs[@]}; do
           runcleanup rm -rf "$x"
         done
-        # Apt removals in foreground, as we can't parallelize these
+        # Apt removals in foreground, as we can't parallelize these.
+        # Only attempt removal if matching packages are installed.
         for x in ${unwanted_pkgs[@]}; do
-          /bin/time -f '%E %C' sudo apt-get remove -y $x
+          if dpkg -l "$x" >/dev/null 2>&1; then
+            /bin/time -f '%E %C' sudo apt-get remove -y "$x"
+          fi
         done
     # We really want support for heredocs
     - name: Update podman and install just

devenv/.dockerignore

@@ -15,3 +15,5 @@
 !fetch-tools.sh
 !install-rust.sh
 !install-kani.sh
+!devenv-selftest.sh
+!userns-setup

devenv/Containerfile.c10s

@@ -1,7 +1,7 @@
 # These aren't packages, just low-dependency binaries dropped in /usr/local/bin
 # so we can fetch them independently in a separate build.
 ARG base=quay.io/centos/centos:stream10
 FROM $base as base
 # Life is too short to care about dash
 RUN ln -sfr /bin/bash /bin/sh
 RUN <<EORUN
@@ -25,7 +25,7 @@
 dnf -y makecache
 EORUN
 
 FROM base as tools
 # renovate: datasource=github-releases depName=bootc-dev/bcvk
 ARG bcvkversion=v0.9.0
 # renovate: datasource=github-releases depName=ossf/scorecard
@@ -33,12 +33,12 @@
 COPY fetch-tools.sh /run/src/
 RUN bcvkversion=$bcvkversion scorecardversion=$scorecardversion /run/src/fetch-tools.sh
 
 FROM base as rust
 COPY install-rust.sh /run/src/
 RUN /run/src/install-rust.sh
 
 # Kani formal verification tool - requires rustup for toolchain management
 FROM rust as kani
 # renovate: datasource=crate depName=kani-verifier
 ARG kaniversion=0.67.0
 RUN dnf install -y gcc && dnf clean all
@@ -55,6 +55,10 @@
 grep -hEve '^#' packages-common.txt packages-c10s.txt | /bin/time -f '%E %C' xargs dnf -y install
 grep -vEe '^#' build-deps-c10s.txt | /bin/time -f '%E %C' xargs dnf -y builddep
 dnf clean all
+# Restore file capabilities for newuidmap/newgidmap - these are defined in the
+# shadow-utils RPM but get stripped during container image builds.
+# Required for nested rootless podman.
+rpm --setcaps shadow-utils
 EORUN
 COPY npm.txt /run/src
 RUN grep -vEe '^#' npm.txt | /bin/time -f '%E %C' xargs npm i -g
@@ -71,6 +75,13 @@
 ENV KANI_HOME=/usr/local/kani
 # Setup for codespaces
 COPY devenv-init.sh /usr/local/bin/
+COPY userns-setup /usr/lib/devenv/userns-setup
+COPY devenv-selftest.sh /usr/libexec/
+# Set file capabilities for newuidmap/newgidmap (C10s shadow-utils doesn't set these by default,
+# unlike Debian's uidmap package). Required for nested rootless podman.
+RUN chmod 755 /usr/libexec/devenv-selftest.sh /usr/lib/devenv/userns-setup && \
+    setcap cap_setuid+ep /usr/bin/newuidmap && \
+    setcap cap_setgid+ep /usr/bin/newgidmap
 
 WORKDIR /
 # Create user before declaring volumes so home directory has correct ownership
@@ -81,6 +92,8 @@
 mkdir -p ~devenv/.local/share/containers
 chown -R -h devenv: ~devenv/.local
 echo 'devenv ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/devenv && chmod 0440 /etc/sudoers.d/devenv
+# TODO: /etc/shadow permissions need fixing for PAM/sudo with --userns=keep-id
+# See https://github.com/bootc-dev/infra/issues/XXX
 EORUN
 # To avoid overlay-on-overlay with nested containers
 VOLUME [ "/var/lib/containers", "/home/devenv/.local/share/containers/" ]

devenv/Containerfile.debian

@@ -1,7 +1,7 @@
 # These aren't packages, just low-dependency binaries dropped in /usr/local/bin
 # so we can fetch them independently in a separate build.
 ARG base=docker.io/library/debian:sid
 FROM $base as base
 # Life is too short to care about dash
 RUN ln -sfr /bin/bash /bin/sh
 RUN <<EORUN
@@ -25,7 +25,7 @@
 apt -y update
 EORUN
 
 FROM base as tools
 # renovate: datasource=github-releases depName=bootc-dev/bcvk
 ARG bcvkversion=v0.9.0
 # renovate: datasource=github-releases depName=ossf/scorecard
@@ -33,12 +33,12 @@
 COPY fetch-tools.sh /run/src/
 RUN bcvkversion=$bcvkversion scorecardversion=$scorecardversion /run/src/fetch-tools.sh
 
 FROM base as rust
 COPY install-rust.sh /run/src/
 RUN /run/src/install-rust.sh
 
 # Kani formal verification tool - requires rustup for toolchain management
 FROM rust as kani
 # renovate: datasource=crate depName=kani-verifier
 ARG kaniversion=0.67.0
 RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev && rm -rf /var/lib/apt/lists/*
@@ -71,6 +71,9 @@
 ENV KANI_HOME=/usr/local/kani
 # Setup for codespaces
 COPY devenv-init.sh /usr/local/bin/
+COPY userns-setup /usr/lib/devenv/userns-setup
+COPY devenv-selftest.sh /usr/libexec/
+RUN chmod 755 /usr/libexec/devenv-selftest.sh /usr/lib/devenv/userns-setup
 
 WORKDIR /
 # Create user before declaring volumes so home directory has correct ownership

devenv/devenv-init.sh

@@ -1,26 +1,3 @@
 #!/bin/bash
-set -euo pipefail
-# Set things up so that podman can run nested inside the privileged
-# docker container of a codespace or devpod.
-
-# Fix the propagation - only needed in some environments (e.g., codespaces)
-# In devpod with rootless podman, / may already have shared propagation
-# or we may not have permission to remount it.
-propagation=$(findmnt -J -o TARGET,PROPAGATION / | jq -r '.filesystems[0].propagation // "unknown"')
-if [ "$propagation" = "private" ]; then
-    if mount -o remount --make-shared / 2>/dev/null; then
-        echo "Set / to shared propagation"
-    else
-        echo "Warning: Could not set / to shared propagation (may not be needed)"
-    fi
-fi
-
-# This is actually safe to expose to all users really, like Fedora derivatives do
-if [ -e /dev/kvm ]; then
-    chmod a+rw /dev/kvm 2>/dev/null || true
-fi
-
-# Handle nested cgroups - update containers.conf if it exists and has the settings commented out
-if [ -f /usr/share/containers/containers.conf ]; then
-    sed -i -e 's,^#cgroups =.*,cgroups = "no-conmon",' -e 's,^#cgroup_manager =.*,cgroup_manager = "cgroupfs",' /usr/share/containers/containers.conf
-fi
+# Thin wrapper that calls the Python implementation
+exec python3 /usr/lib/devenv/userns-setup "$@"

devenv/devenv-selftest.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Test that nested podman and VMs work correctly in this devcontainer.
+# This script is designed to be run inside the container after devenv-init.sh
+# has already been executed (e.g., via postCreateCommand).
+set -euo pipefail
+
+echo "=== Testing nested podman and VMs ==="
+
+echo "Podman version:"
+podman --version
+
+echo "Podman info (rootless):"
+podman info --format '{{.Host.Security.Rootless}}'
+
+# Use CentOS Stream 10 as the test image for both container and VM
+image="quay.io/centos-bootc/centos-bootc:stream10"
+
+echo "Pulling $image..."
+podman pull "$image"
+
+echo "Running nested container..."
+podman run --rm "$image" echo "Hello from nested podman!"
+
+echo "=== Nested container test passed ==="
+
+# Test bcvk (VM) if available and /dev/kvm exists
+if command -v bcvk >/dev/null 2>&1 && [ -e /dev/kvm ]; then
+    echo ""
+    echo "=== Testing bcvk VM ==="
+    echo "bcvk version:"
+    bcvk --version
+
+    echo "Running bcvk ephemeral VM with SSH..."
+    bcvk ephemeral run-ssh "$image" -- echo "Hello from bcvk VM!"
+
+    echo "=== bcvk VM test passed ==="
+else
+    echo ""
+    echo "=== Skipping bcvk VM test (bcvk not available or /dev/kvm missing) ==="
+fi
+
+echo ""
+echo "=== All tests passed ==="