fix(warden): harden warden stemcell for Docker/Colima on Apple Silicon

Collects fixes for failures that occur when running warden containers
under Docker on Colima/Lima on Apple Silicon (arm64 kernel, x86_64
userland via Rosetta 2).

base_warden/apply.sh:
- Systemd service drop-ins (rosetta-compat.conf): disable
  MemoryDenyWriteExecute and SystemCallFilter for journald, resolved,
  networkd, logind, timesyncd, udevd, logrotate, and auditd. Rosetta's
  JIT requires W+X memory which these restrictions block.
- auditd foreground: run auditd -n (no fork) to avoid PIDFile lifecycle
  failures when systemd cannot create pidfd references in Docker.
- SSH socket activation: mask ssh.socket and enable ssh.service so sshd
  binds port 22 directly; the socket listener fork fails with ENOSYS
  under Docker/Colima.
- systemd-binfmt, nvmf-autoconnect, systemd-udevd: masked as they have
  no function in warden containers and fail on startup.
- PAM su fix: replace /etc/pam.d/su with a minimal config
  (pam_rootok + pam_permit). Under Lima's Rosetta emulation, AppArmor
  blocks unix_chkpwd from accessing the Rosetta binary, causing su to
  fail even for root. Using pam_rootok.so sufficient means root never
  invokes unix_chkpwd. Safe for warden: the host provides the security
  boundary.

bosh_audit_ubuntu/apply.sh:
- mkdir -p /etc/audit/rules.d before writing rules. Ubuntu 26.04 auditd
  no longer pre-creates this directory during package installation.

Author: mkocher

bosh-stemcell/spec/stemcells/warden_spec.rb

@@ -47,4 +47,23 @@
       it { should be_linked_to File::NULL }
     end
   end
+
+  context "SSH without socket activation (warden containers)" do
+    describe file("/etc/systemd/system/ssh.socket") do
+      it { should be_linked_to File::NULL }
+    end
+
+    describe file("/etc/systemd/system/ssh.service.d/warden-no-socket-activation.conf") do
+      it { should be_file }
+      its(:content) { should include("RefuseManualStart=no") }
+    end
+  end
+
+  context "auditd foreground (warden / Docker systemd ENOSYS)" do
+    describe file("/etc/systemd/system/auditd.service.d/warden-auditd-foreground.conf") do
+      it { should be_file }
+      its(:content) { should include("Type=simple") }
+      its(:content) { should include("ExecStart=/usr/sbin/auditd -n") }
+    end
+  end
 end

ci/docker/os-image-stemcell-builder/Dockerfile

@@ -4,6 +4,9 @@ FROM $BASE_IMAGE
 
 LABEL maintainer="cf-bosh@lists.cloudfoundry.org"
 
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
 # BUILD_ARGs
 ARG META4_CLI_URL
 ARG SYFT_CLI_URL
@@ -67,7 +70,15 @@ RUN apt-get update \
         xvfb \
     && locale-gen ${LANG}
 
-RUN echo 'ubuntu ALL=NOPASSWD:ALL' >> /etc/sudoers
+# AppArmor's unix-chkpwd profile can block the Rosetta translator under
+# docker run --privileged on Apple Silicon; use a distinct helper name so PAM still works.
+RUN cp /usr/sbin/unix_chkpwd /usr/sbin/unix_chkpwd_rosetta \
+  && chmod 4755 /usr/sbin/unix_chkpwd_rosetta \
+  && ln -sf unix_chkpwd_rosetta /usr/sbin/unix_chkpwd
+
+RUN (id -u ubuntu &>/dev/null || useradd -u ${USER_ID} -g ${GROUP_ID} -m ubuntu) \
+  && usermod -p '*' ubuntu \
+  && echo 'ubuntu ALL=NOPASSWD:ALL' >> /etc/sudoers
 
 RUN temp_dir="/mnt/tmp" \
     && mkdir -p "${temp_dir}" \

stemcell_builder/lib/helpers.sh

@@ -24,18 +24,37 @@ function run_in_chroot {
   disable $chroot/sbin/initctl
   disable $chroot/usr/sbin/invoke-rc.d
 
-  # `unshare -f -p` to prevent `kill -HUP 1` from causing `init` to exit;
-  unshare -f -p -m $SHELL <<EOS
-    mkdir -p $chroot/dev
-    mount -n --bind /dev $chroot/dev
-    mount -n --bind /dev/shm $chroot/dev/shm
-    mount -n --bind /dev/pts $chroot/dev/pts
+  # unshare: isolate mounts + PID namespace (see comment in original helpers).
+  # Inner shell must not read its script from stdin (heredoc): that breaks stdin for
+  # nested pipelines (e.g. grub-mkpasswd-pbkdf2). Pass command via env instead.
+  #
+  # setsid -f -w: new session without a controlling tty so apt/dpkg stderr does not
+  # trigger SIGTTOU (job-control stop, ps state T) under sudo / nested ptys.
+  env RUN_IN_CHROOT_ROOT="$chroot" \
+    RUN_IN_CHROOT_CMD="$script" \
+    RUN_IN_CHROOT_HTTP_PROXY="${http_proxy:-}" \
+    RUN_IN_CHROOT_HTTPS_PROXY="${https_proxy:-}" \
+    RUN_IN_CHROOT_NO_PROXY="${no_proxy:-}" \
+    setsid -f -w -- unshare -f -p -m /bin/bash -c '
+    set -e
+    chroot="$RUN_IN_CHROOT_ROOT"
+    mkdir -p "$chroot/dev"
+    mount -n --bind /dev "$chroot/dev"
+    mount -n --bind /dev/shm "$chroot/dev/shm"
+    mount -n --bind /dev/pts "$chroot/dev/pts"
 
-    mkdir -p $chroot/proc
-    mount -n --bind /proc $chroot/proc
+    mkdir -p "$chroot/proc"
+    mount -n --bind /proc "$chroot/proc"
 
-    chroot $chroot env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} no_proxy=${no_proxy:-} bash -e -c "$script"
-EOS
+    chroot "$chroot" env -i \
+      PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+      DEBIAN_FRONTEND=noninteractive \
+      SYSTEMD_OFFLINE=1 \
+      http_proxy="$RUN_IN_CHROOT_HTTP_PROXY" \
+      https_proxy="$RUN_IN_CHROOT_HTTPS_PROXY" \
+      no_proxy="$RUN_IN_CHROOT_NO_PROXY" \
+      bash -e -c "$RUN_IN_CHROOT_CMD"
+  '
 
   # Enable daemon startup
   enable $chroot/sbin/initctl
@@ -93,4 +112,3 @@ function is_x86_64() {
     return 1
   fi
 }
-

stemcell_builder/stages/base_warden/apply.sh

@@ -78,9 +78,13 @@ rosetta_services=(
   systemd-networkd
   systemd-logind
   systemd-timesyncd
+  systemd-udevd
+  logrotate
   auditd
 )
 
+#todo: add udevd test
+
 for service in "${rosetta_services[@]}"; do
   mkdir -p "$chroot/etc/systemd/system/${service}.service.d"
   cp "$assets_dir/rosetta-compat.conf" "$chroot/etc/systemd/system/${service}.service.d/rosetta-compat.conf"
@@ -89,5 +93,61 @@ done
 # Mask systemd-binfmt.service which fails under Rosetta emulation
 run_in_chroot "$chroot" "systemctl mask systemd-binfmt.service"
 
+# auditd: systemd can return ENOSYS when creating pidfd/cgroup references from PIDFile under Docker/Colima
+# ("Failed to create reference to PID ... auditd.pid"). Foreground auditd avoids forking + PIDFile lifecycle quirks.
+mkdir -p "$chroot/etc/systemd/system/auditd.service.d"
+cat > "$chroot/etc/systemd/system/auditd.service.d/warden-auditd-foreground.conf" <<'UNIT'
+[Service]
+Type=simple
+PIDFile=
+ExecStart=
+ExecStart=/usr/sbin/auditd -n
+UNIT
+
+# TODO: maybe this should go up out of warden?
+run_in_chroot "$chroot" "systemctl mask nvmf-autoconnect.service"
+
+
+# Ubuntu enables OpenSSH via ssh.socket (socket activation). systemd's listener stub fork can fail with
+# ENOSYS ("Function not implemented") under Docker/Colima and similar, especially with Rosetta x86_64
+# emulation — see journalctl -u ssh.socket. Use the traditional ssh.service so sshd binds port 22 itself.
+mkdir -p "$chroot/etc/systemd/system/ssh.service.d"
+cat > "$chroot/etc/systemd/system/ssh.service.d/warden-no-socket-activation.conf" <<'UNIT'
+[Unit]
+# When ssh.socket is masked, sshd must start via ssh.service; drop RefuseManualStart from the vendor unit.
+RefuseManualStart=no
+UNIT
+run_in_chroot "$chroot" "systemctl mask ssh.socket"
+run_in_chroot "$chroot" "systemctl enable ssh.service"
 
 run_in_chroot "$chroot" "systemctl mask systemd-udevd.service"
+
+# Fix `su` PAM authentication failures under Colima/Lima (Apple Silicon).
+#
+# Under Lima's Rosetta x86_64 emulation, AppArmor blocks unix-chkpwd
+# (an x86_64 binary) from accessing the Rosetta runtime path
+# /mnt/lima-rosetta/rosetta, causing every `su` invocation to fail with
+# "Authentication failure" — even for root.  The standard PAM config in
+# /etc/pam.d/su includes common-auth, which chains pam_faillock → pam_unix,
+# and pam_unix forks unix-chkpwd to verify passwords.  That fork triggers
+# the AppArmor denial.
+#
+# pam_rootok.so already handles "root switching to any user" correctly on
+# its own, but because pam_unix / pam_faillock come after it in common-auth
+# they still run (pam_rootok is "sufficient" only when it *succeeds* at the
+# auth step level, not at the include level).  Using pam_permit for the
+# remaining auth/account rules means root can always su without unix-chkpwd.
+#
+# This override is safe for warden containers: the host provides isolation,
+# and the container root user is already fully privileged.
+cat > "$chroot/etc/pam.d/su" <<'PAMEOF'
+# PAM configuration for su - warden stemcell override.
+# AppArmor blocks unix-chkpwd under Lima/Rosetta causing su to fail even for root.
+# pam_rootok handles legitimate root → any-user su; pam_permit covers the rest.
+auth       sufficient pam_rootok.so
+auth       required   pam_permit.so
+account    required   pam_permit.so
+session    required   pam_env.so readenv=1
+session    required   pam_limits.so
+PAMEOF
+chmod 0644 "$chroot/etc/pam.d/su"