Skip to content

Commit d2bdca0

Browse files
authored
sandlock-oci: single-sandbox OCI exec via an in-process PID-1 (#110)
Add the OCI `exec` verb to the sandlock-oci runtime: run additional processes inside an already-running sandbox that share the SAME confinement (Landlock ruleset, seccomp-notify supervisor, and chroot) as the main workload. This needs an in-sandbox PID-1 that forks the workload and any exec'd siblings on demand: a Landlock domain can only be joined by a process forked or exec'd from within it (there is no setns into Landlock), so a process can be placed into the running sandbox only by forking it from inside. The CLI relays the OCI verbs (create/start/exec/kill/delete/state/checkpoint) to that PID-1 over a newline-delimited JSON control socket; `exec` passes the child's stdio over SCM_RIGHTS so the exec'd process lands inside the chroot under the shared sandbox and its exit status is reported back. The PID-1 runs IN-PROCESS, not as a separately exec'd binary. The supervisor's confined child is already a fork() of the supervisor, so its code is mapped; after confinement is installed it runs the control loop (run_init) directly instead of execve. Because nothing is exec'd for the PID-1 itself, Landlock has no execve to authorize. Workloads it forks exec real files inside the rootfs via the normal path-authorization path (open-in-root plus fd injection), which the chroot gate and the Landlock ruleset allow. Core mechanism (sandlock-core): Sandbox::create_with_in_child_main plus ChildEntry::{Exec, InProcess} models the confined child's terminal action after confinement as either execve a command or run a fn() in-process. The in-process arm sets the process name via prctl(PR_SET_NAME) so the PID-1 shows as `sandlock-init` rather than the supervisor. This supersedes an earlier memfd-based design (embed a static sandlock-init, stage it in a memfd, launch it with execveat(AT_EMPTY_PATH)). Executing a path-less memfd is denied by Landlock on conformant kernels, so that design failed the OCI exec tests in CI from the start; the in-process PID-1 is kernel-version-independent. As a result this also removes: - the separate sandlock-init binary and crate (folded into sandlock-oci as a module providing run_init plus the control protocol) - the build.rs memfd embed and the musl-static release/CI build - the now-unused exec-from-fd machinery: Sandbox.exec_fd and the chroot AT_EMPTY_PATH execve passthrough. A chroot'd fd-exec now resolves the fd's real target via /proc, so a workload may fexecve a file inside the rootfs but a memfd resolves outside it and correctly gets EACCES. Tested: end-to-end OCI exec, checkpoint-of-running-container, and process-group-stop integration tests on x86_64 and aarch64; `sandlock-oci exec` verified to run a process inside a running container's rootfs under the shared sandbox. Signed-off-by: Cong Wang <cwang@multikernel.io>
1 parent 708fa47 commit d2bdca0

11 files changed

Lines changed: 1448 additions & 79 deletions

File tree

crates/sandlock-core/src/context.rs

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// Fork + confinement sequence: child-side Landlock + seccomp application
22
// and parent-child pipe synchronization.
33

4-
use std::ffi::CString;
4+
use std::ffi::{CStr, CString};
55
use std::io;
66
use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd};
77

@@ -193,9 +193,23 @@ fn write_id_maps(real_uid: u32, real_gid: u32, target_uid: u32, target_gid: u32)
193193
/// handlers). Lifetimes tie everything to the parent's stack frame — the
194194
/// child never outlives the fork point because `confine_child` either execs
195195
/// or exits.
196+
/// The terminal action `confine_child` performs after confinement is installed.
197+
/// Exactly one of the two: there is no longer a "command plus optional override".
198+
pub(crate) enum ChildEntry<'a> {
199+
/// `execve` this command (the normal path). argv[0] becomes the process name.
200+
Exec(&'a [CString]),
201+
/// Run this function in-process, with the process named `name`. Used for the
202+
/// OCI in-sandbox PID-1: the child is a fork of the supervisor so the code is
203+
/// already mapped, nothing is exec'd, and Landlock has no execve to
204+
/// authorize. `run` must not return; `confine_child` `_exit(0)`s if it does.
205+
InProcess { name: &'a CStr, run: fn() },
206+
}
207+
196208
pub(crate) struct ChildSpawnArgs<'a> {
197209
pub sandbox: &'a Sandbox,
198-
pub cmd: &'a [CString],
210+
/// Terminal action after confinement: `execve` a command or run a fn
211+
/// in-process. See [`ChildEntry`].
212+
pub entry: ChildEntry<'a>,
199213
pub pipes: &'a PipePair,
200214
/// Skip the user-notification supervisor: child installs a kernel-only
201215
/// deny filter, parent reads `notif_fd_num = 0` and never starts a
@@ -215,14 +229,22 @@ pub(crate) struct ChildSpawnArgs<'a> {
215229
pub parent_pid: libc::pid_t,
216230
}
217231

218-
/// Apply irreversible confinement (Landlock + seccomp) then exec the command.
232+
/// Set the calling thread/process name (`/proc/<pid>/comm`, shown by `ps`). The
233+
/// kernel truncates to 15 bytes + NUL. Used for the in-process PID-1, which has
234+
/// no `execve` to set its name from argv[0].
235+
fn set_proc_name(name: &CStr) {
236+
unsafe { libc::prctl(libc::PR_SET_NAME, name.as_ptr() as libc::c_ulong, 0, 0, 0) };
237+
}
238+
239+
/// Apply irreversible confinement (Landlock + seccomp), then either `execve` the
240+
/// command or run an in-process entrypoint, per [`ChildEntry`].
219241
///
220-
/// This function **never returns**: it calls `execvp` on success or
221-
/// `_exit(127)` on any error.
242+
/// This function **never returns**: on success it execs or runs the entrypoint
243+
/// (which `_exit`s); on any error it `_exit(127)`s.
222244
pub(crate) fn confine_child(args: ChildSpawnArgs<'_>) -> ! {
223245
let ChildSpawnArgs {
224246
sandbox,
225-
cmd,
247+
entry,
226248
pipes,
227249
no_supervisor,
228250
keep_fds,
@@ -490,6 +512,22 @@ pub(crate) fn confine_child(args: ChildSpawnArgs<'_>) -> ! {
490512
// Empty list = all GPUs visible, don't set env vars
491513
}
492514

515+
// 14. Terminal action: run the in-process entrypoint, or fall through to
516+
// execve the command. The in-process arm diverges (`_exit`), so the match
517+
// yields the command slice only on the `Exec` path.
518+
let cmd: &[CString] = match entry {
519+
ChildEntry::InProcess { name, run } => {
520+
// Name the PID-1 so ps / /proc/<pid>/comm read correctly: there is
521+
// no execve here to set argv[0]. The child is a fork of the
522+
// supervisor, so `run`'s code is already mapped; running it directly
523+
// avoids an execve that Landlock would otherwise have to authorize.
524+
set_proc_name(name);
525+
run();
526+
unsafe { libc::_exit(0) };
527+
}
528+
ChildEntry::Exec(cmd) => cmd,
529+
};
530+
493531
// 14. exec
494532
debug_assert!(!cmd.is_empty(), "cmd must not be empty");
495533
let argv_ptrs: Vec<*const libc::c_char> = cmd

crates/sandlock-core/src/sandbox.rs

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,17 @@ pub struct Sandbox {
337337

338338
// Environment
339339
pub chroot: Option<PathBuf>,
340+
341+
/// When set, the confined child runs this function in-process instead of
342+
/// `execve`-ing a workload. Used to run an in-sandbox PID-1 (the OCI
343+
/// `sandlock-init` control loop) without exec'ing a separate image: the
344+
/// child is already a fork of the supervisor, so its code is mapped, and
345+
/// because nothing is exec'd, Landlock has no execution to authorize. The
346+
/// function must not return (it loops and `_exit`s); `confine_child` calls
347+
/// `_exit(0)` if it does.
348+
#[serde(skip)]
349+
pub in_child_main: Option<fn()>,
350+
340351
pub clean_env: bool,
341352
pub env: HashMap<String, String>,
342353
// Devices
@@ -444,6 +455,7 @@ impl Clone for Sandbox {
444455
on_error: self.on_error.clone(),
445456
fs_mount: self.fs_mount.clone(),
446457
chroot: self.chroot.clone(),
458+
in_child_main: self.in_child_main,
447459
clean_env: self.clean_env,
448460
env: self.env.clone(),
449461
gpu_devices: self.gpu_devices.clone(),
@@ -847,6 +859,28 @@ impl Sandbox {
847859
self.do_create(cmd, false).await
848860
}
849861

862+
/// Create a confined child that, instead of `execve`-ing a workload, runs
863+
/// `entrypoint` in-process after confinement is installed. The child is a
864+
/// `fork()` of this process, so `entrypoint`'s code is already mapped; no
865+
/// image is exec'd, so Landlock has nothing to authorize for the child's own
866+
/// startup. `extra_fds` maps caller fds onto fixed fd numbers in the child
867+
/// (e.g. the control channel). Used to run the OCI in-sandbox PID-1.
868+
///
869+
/// `name` is not exec'd; it sets the child's process name
870+
/// (`/proc/<pid>/comm`). `start()` releases the parked child to run
871+
/// `entrypoint`.
872+
pub async fn create_with_in_child_main(
873+
&mut self,
874+
name: &str,
875+
extra_fds: Vec<(i32, i32)>,
876+
entrypoint: fn(),
877+
) -> Result<(), crate::error::SandlockError> {
878+
self.ensure_runtime()?;
879+
self.in_child_main = Some(entrypoint);
880+
self.rt_mut().extra_fds = extra_fds;
881+
self.do_create(&[name], false).await
882+
}
883+
850884
/// Freeze the sandbox: hold fork notifications + SIGSTOP the process group.
851885
pub(crate) async fn freeze(&self) -> Result<(), crate::error::SandlockError> {
852886
use crate::error::{SandboxRuntimeError, SandlockError};
@@ -880,8 +914,20 @@ impl Sandbox {
880914
let pid = self.runtime.as_ref()
881915
.and_then(|rt| rt.child_pid)
882916
.ok_or(SandlockError::Runtime(SandboxRuntimeError::NotRunning))?;
917+
self.checkpoint_pid(pid).await
918+
}
919+
920+
/// Capture a checkpoint targeting a specific pid instead of the sandbox's
921+
/// direct child. The target must be a fork-descendant confined by the same
922+
/// policy (e.g. the workload spawned by sandlock-init). `target_pid` must
923+
/// be positive.
924+
pub async fn checkpoint_pid(&self, target_pid: i32) -> Result<crate::checkpoint::Checkpoint, crate::error::SandlockError> {
925+
use crate::error::{SandboxRuntimeError, SandlockError};
926+
if target_pid <= 0 {
927+
return Err(SandlockError::Runtime(SandboxRuntimeError::NotRunning));
928+
}
883929
self.freeze().await?;
884-
let cp = crate::checkpoint::capture(pid, self);
930+
let cp = crate::checkpoint::capture(target_pid, self);
885931
self.thaw().await?;
886932
cp
887933
}
@@ -1454,9 +1500,15 @@ impl Sandbox {
14541500
.collect();
14551501

14561502
let sandbox_name = self.rt().name.clone();
1503+
// In-process entrypoint (OCI PID-1) names the process from cmd[0];
1504+
// otherwise execve the command.
1505+
let entry = match self.in_child_main {
1506+
Some(run) => context::ChildEntry::InProcess { name: c_cmd[0].as_c_str(), run },
1507+
None => context::ChildEntry::Exec(&c_cmd),
1508+
};
14571509
context::confine_child(context::ChildSpawnArgs {
14581510
sandbox: self,
1459-
cmd: &c_cmd,
1511+
entry,
14601512
pipes: &pipes,
14611513
no_supervisor,
14621514
keep_fds: &gather_keep_fds,

crates/sandlock-core/src/sandbox/builder.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,7 @@ impl SandboxBuilder {
753753
on_error: self.on_error.unwrap_or_default(),
754754
fs_mount: self.fs_mount,
755755
chroot: self.chroot,
756+
in_child_main: None,
756757
clean_env: self.clean_env,
757758
env: self.env,
758759
gpu_devices: self.gpu_devices,

crates/sandlock-oci/src/fdpass.rs

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
//! SCM_RIGHTS file-descriptor passing over the supervisor control socket.
2+
//!
3+
//! `exec` is the only control command that carries open fds (the CLI's
4+
//! stdin/stdout/stderr, which the container shim wired to the exec stream).
5+
//! The daemon must receive those fds with the SAME `recvmsg` that reads the
6+
//! command bytes, because ancillary data binds to specific bytes. All other
7+
//! commands send an empty ancillary payload, so the receive path is uniform.
8+
9+
use std::io;
10+
use std::os::unix::io::{FromRawFd, OwnedFd, RawFd};
11+
12+
/// Send `data` plus `fds` (as SCM_RIGHTS) over a blocking unix socket.
13+
pub fn send_with_fds(
14+
stream: &std::os::unix::net::UnixStream,
15+
data: &[u8],
16+
fds: &[RawFd],
17+
) -> io::Result<()> {
18+
use std::os::unix::io::AsRawFd;
19+
20+
let mut iov = libc::iovec {
21+
iov_base: data.as_ptr() as *mut libc::c_void,
22+
iov_len: data.len(),
23+
};
24+
let fds_bytes = std::mem::size_of_val(fds) as u32;
25+
let cmsg_space = if fds.is_empty() {
26+
0usize
27+
} else {
28+
unsafe { libc::CMSG_SPACE(fds_bytes) as usize }
29+
};
30+
let mut cmsg_buf = vec![0u8; cmsg_space];
31+
32+
let mut msg: libc::msghdr = unsafe { std::mem::zeroed() };
33+
msg.msg_iov = &mut iov;
34+
msg.msg_iovlen = 1;
35+
if !fds.is_empty() {
36+
msg.msg_control = cmsg_buf.as_mut_ptr() as *mut libc::c_void;
37+
msg.msg_controllen = cmsg_space as _;
38+
unsafe {
39+
let cmsg = libc::CMSG_FIRSTHDR(&msg);
40+
(*cmsg).cmsg_level = libc::SOL_SOCKET;
41+
(*cmsg).cmsg_type = libc::SCM_RIGHTS;
42+
(*cmsg).cmsg_len = libc::CMSG_LEN(fds_bytes) as _;
43+
std::ptr::copy_nonoverlapping(
44+
fds.as_ptr(),
45+
libc::CMSG_DATA(cmsg) as *mut RawFd,
46+
fds.len(),
47+
);
48+
}
49+
}
50+
51+
let n = unsafe { libc::sendmsg(stream.as_raw_fd(), &msg, 0) };
52+
if n < 0 {
53+
return Err(io::Error::last_os_error());
54+
}
55+
Ok(())
56+
}
57+
58+
/// Receive bytes plus up to `max_fds` SCM_RIGHTS fds from a raw socket fd
59+
/// (one `recvmsg`). Returns the data and any received `OwnedFd`s.
60+
pub fn recv_with_fds(fd: RawFd, max_fds: usize) -> io::Result<(Vec<u8>, Vec<OwnedFd>)> {
61+
let mut buf = vec![0u8; 8192];
62+
let mut iov = libc::iovec {
63+
iov_base: buf.as_mut_ptr() as *mut libc::c_void,
64+
iov_len: buf.len(),
65+
};
66+
let cmsg_space =
67+
unsafe { libc::CMSG_SPACE((max_fds * std::mem::size_of::<RawFd>()) as u32) as usize };
68+
let mut cmsg_buf = vec![0u8; cmsg_space];
69+
70+
let mut msg: libc::msghdr = unsafe { std::mem::zeroed() };
71+
msg.msg_iov = &mut iov;
72+
msg.msg_iovlen = 1;
73+
msg.msg_control = cmsg_buf.as_mut_ptr() as *mut libc::c_void;
74+
msg.msg_controllen = cmsg_space as _;
75+
76+
let n = unsafe { libc::recvmsg(fd, &mut msg, 0) };
77+
if n < 0 {
78+
return Err(io::Error::last_os_error());
79+
}
80+
81+
let mut fds = Vec::new();
82+
unsafe {
83+
let mut cmsg = libc::CMSG_FIRSTHDR(&msg);
84+
while !cmsg.is_null() {
85+
if (*cmsg).cmsg_level == libc::SOL_SOCKET && (*cmsg).cmsg_type == libc::SCM_RIGHTS {
86+
let payload = (*cmsg).cmsg_len as usize - libc::CMSG_LEN(0) as usize;
87+
let count = payload / std::mem::size_of::<RawFd>();
88+
let data_ptr = libc::CMSG_DATA(cmsg) as *const RawFd;
89+
for i in 0..count {
90+
fds.push(OwnedFd::from_raw_fd(*data_ptr.add(i)));
91+
}
92+
}
93+
cmsg = libc::CMSG_NXTHDR(&msg, cmsg);
94+
}
95+
}
96+
97+
buf.truncate(n as usize);
98+
Ok((buf, fds))
99+
}
100+
101+
/// Tokio wrapper: await readability, then one `recv_with_fds` on the raw fd.
102+
/// `recvmsg` returns `EAGAIN` (`WouldBlock`) if the socket was not actually
103+
/// ready; loop until it yields data.
104+
pub async fn recv_with_fds_async(
105+
stream: &tokio::net::UnixStream,
106+
max_fds: usize,
107+
) -> io::Result<(Vec<u8>, Vec<OwnedFd>)> {
108+
use std::os::unix::io::AsRawFd;
109+
loop {
110+
stream.readable().await?;
111+
match stream.try_io(tokio::io::Interest::READABLE, || {
112+
recv_with_fds(stream.as_raw_fd(), max_fds)
113+
}) {
114+
Ok(res) => return Ok(res),
115+
Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => continue,
116+
Err(e) => return Err(e),
117+
}
118+
}
119+
}
120+
121+
#[cfg(test)]
122+
mod tests {
123+
use super::*;
124+
use std::io::Write;
125+
use std::os::unix::io::AsRawFd;
126+
use std::os::unix::net::UnixStream;
127+
128+
/// Send a pipe's write-end across a socketpair, then prove the received fd
129+
/// is the SAME open file: writing through it is readable from the original
130+
/// read-end.
131+
#[test]
132+
fn send_and_recv_one_fd_roundtrip() {
133+
let (a, b) = UnixStream::pair().unwrap();
134+
135+
// A pipe whose write end we will pass over the socket.
136+
let mut fds = [0i32; 2];
137+
assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0);
138+
let (pipe_r, pipe_w) = (fds[0], fds[1]);
139+
140+
send_with_fds(&a, b"PING", &[pipe_w]).unwrap();
141+
let (data, got) = recv_with_fds(b.as_raw_fd(), 3).unwrap();
142+
143+
assert_eq!(&data, b"PING");
144+
assert_eq!(got.len(), 1);
145+
146+
// Write through the RECEIVED fd, read from the original pipe read end.
147+
let received_w = got[0].as_raw_fd();
148+
assert_eq!(unsafe { libc::write(received_w, b"Z".as_ptr() as *const _, 1) }, 1);
149+
let mut buf = [0u8; 1];
150+
assert_eq!(unsafe { libc::read(pipe_r, buf.as_mut_ptr() as *mut _, 1) }, 1);
151+
assert_eq!(buf[0], b'Z');
152+
153+
unsafe { libc::close(pipe_r); libc::close(pipe_w); }
154+
let _ = (a, b);
155+
}
156+
157+
#[test]
158+
fn recv_without_fds_returns_empty_vec() {
159+
let (mut a, b) = UnixStream::pair().unwrap();
160+
a.write_all(b"hello").unwrap();
161+
let (data, got) = recv_with_fds(b.as_raw_fd(), 3).unwrap();
162+
assert_eq!(&data, b"hello");
163+
assert!(got.is_empty());
164+
}
165+
}
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
use std::os::unix::io::RawFd;
2+
3+
/// Blocking recvmsg of one message plus up to `max_fds` SCM_RIGHTS fds.
4+
/// Returns (bytes, fds). Empty bytes means EOF/peer closed.
5+
pub fn recv(fd: RawFd, max_fds: usize) -> std::io::Result<(Vec<u8>, Vec<RawFd>)> {
6+
let mut buf = vec![0u8; 65536];
7+
let mut iov = libc::iovec { iov_base: buf.as_mut_ptr() as *mut _, iov_len: buf.len() };
8+
let space = unsafe { libc::CMSG_SPACE((max_fds * 4) as u32) as usize };
9+
let mut cbuf = vec![0u8; space];
10+
let mut msg: libc::msghdr = unsafe { std::mem::zeroed() };
11+
msg.msg_iov = &mut iov;
12+
msg.msg_iovlen = 1;
13+
msg.msg_control = cbuf.as_mut_ptr() as *mut _;
14+
msg.msg_controllen = space as _;
15+
let n = unsafe { libc::recvmsg(fd, &mut msg, 0) };
16+
if n < 0 { return Err(std::io::Error::last_os_error()); }
17+
let mut fds = Vec::new();
18+
unsafe {
19+
let mut c = libc::CMSG_FIRSTHDR(&msg);
20+
while !c.is_null() {
21+
if (*c).cmsg_level == libc::SOL_SOCKET && (*c).cmsg_type == libc::SCM_RIGHTS {
22+
let payload = (*c).cmsg_len as usize - libc::CMSG_LEN(0) as usize;
23+
let p = libc::CMSG_DATA(c) as *const RawFd;
24+
for i in 0..(payload / 4) { fds.push(*p.add(i)); }
25+
}
26+
c = libc::CMSG_NXTHDR(&msg, c);
27+
}
28+
}
29+
buf.truncate(n as usize);
30+
Ok((buf, fds))
31+
}

0 commit comments

Comments
 (0)