Skip to content

Commit adb8116

Browse files
author
Roy Lin
committed
fix(shim): kubectl exec works end-to-end on RuntimeClass=a3s-box
Makes the containerd-shim-a3s-box-v2 path actually serve `kubectl exec` against libkrun MicroVMs, plus the v2.6.0 network/block-IO stats feature. containerd shim: - Stage container env in a rootfs file (BOX_EXEC_ENV_FILE) instead of inlining it: Kubernetes injects ~150 service env vars per pod, which overflow the guest kernel cmdline (COMMAND_LINE_SIZE) and silently break boot. Only a small pointer rides the cmdline now. - Gate Task.wait()'s inspect-poll on the box being confirmed running: containerd calls Wait() concurrently with (and before) Start finishes, during which `inspect` returns "No such container"; treating that as a terminal exit made containerd kill the box it was still starting. - Launch the box as a transient systemd unit so it runs under a clean rlimit/memlock context (libkrun must mlock guest RAM for KVM). - Hold the CRI stdout/stderr FIFO write ends open so the task persists. Also includes the network + block-IO stats feature for `a3s-box stats/top/ps` (net_stats_path threaded through vmm/netproxy/passt). Validated: kubectl exec end-to-end 4/4 on the production cluster; full `src` workspace lib tests green (cargo test --workspace --lib).
1 parent 8036c89 commit adb8116

16 files changed

Lines changed: 1328 additions & 185 deletions

File tree

containerd-shim/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ containerd-shim-protos = { version = "0.11", features = ["async"] }
2121
async-trait = "0.1"
2222
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "sync", "io-util", "fs", "time"] }
2323
log = "0.4"
24+
libc = "0.2"
2425
serde = { version = "1", features = ["derive"] }
2526
serde_json = "1"
2627

containerd-shim/src/logic.rs

Lines changed: 38 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -39,18 +39,31 @@ pub fn parse_spec(spec: &Value) -> ParsedSpec {
3939
.and_then(|v| v.as_u64())
4040
.map(|b| (b / (1024 * 1024)).max(1));
4141
let cpus = match (
42-
spec.pointer("/linux/resources/cpu/quota").and_then(|v| v.as_i64()),
43-
spec.pointer("/linux/resources/cpu/period").and_then(|v| v.as_u64()),
42+
spec.pointer("/linux/resources/cpu/quota")
43+
.and_then(|v| v.as_i64()),
44+
spec.pointer("/linux/resources/cpu/period")
45+
.and_then(|v| v.as_u64()),
4446
) {
4547
(Some(q), Some(p)) if q > 0 && p > 0 => Some(((q as f64 / p as f64).ceil()) as u32),
4648
_ => None,
4749
};
48-
ParsedSpec { is_sandbox, image, args, env, cpus, memory_mb }
50+
ParsedSpec {
51+
is_sandbox,
52+
image,
53+
args,
54+
env,
55+
cpus,
56+
memory_mb,
57+
}
4958
}
5059

5160
fn string_array(v: Option<&Value>) -> Vec<String> {
5261
v.and_then(|a| a.as_array())
53-
.map(|a| a.iter().filter_map(|x| x.as_str().map(String::from)).collect())
62+
.map(|a| {
63+
a.iter()
64+
.filter_map(|x| x.as_str().map(String::from))
65+
.collect()
66+
})
5467
.unwrap_or_default()
5568
}
5669

@@ -63,12 +76,12 @@ pub fn run_args(
6376
env: &[String],
6477
cmd: &[String],
6578
) -> Vec<String> {
66-
let mut v = vec![
67-
"run".to_string(),
68-
"-d".to_string(),
69-
"--name".to_string(),
70-
id.to_string(),
71-
];
79+
// Foreground `run` (no -d). This is launched as the main process of a transient
80+
// systemd unit (see service.rs): foreground run blocks until the box exits, so
81+
// the unit stays active for the container's whole life. A detached `run -d`
82+
// would exit right after boot, the unit would deactivate, and that exit gets
83+
// reported as the container exiting — tearing the task down seconds after start.
84+
let mut v = vec!["run".to_string(), "--name".to_string(), id.to_string()];
7285
if let Some(c) = cpus {
7386
v.push("--cpus".to_string());
7487
v.push(c.to_string());
@@ -120,21 +133,6 @@ pub fn a3s_home() -> String {
120133
std::env::var("A3S_HOME").unwrap_or_else(|_| "/var/lib/a3s-box".to_string())
121134
}
122135

123-
/// Numeric box status -> name (mirrors the protobuf Status enum we report).
124-
pub fn status_name(s: i32) -> &'static str {
125-
match s {
126-
1 => "created",
127-
2 => "running",
128-
3 => "stopped",
129-
_ => "unknown",
130-
}
131-
}
132-
133-
/// Parse an exit code printed by `a3s-box wait` (its stdout is the integer code).
134-
pub fn parse_wait_exit_code(stdout: &str) -> u32 {
135-
stdout.trim().parse::<i32>().unwrap_or(0) as u32
136-
}
137-
138136
#[cfg(test)]
139137
mod tests {
140138
use super::*;
@@ -199,7 +197,7 @@ mod tests {
199197
fn run_args_minimal() {
200198
assert_eq!(
201199
run_args("box1", "alpine", None, None, &[], &[]),
202-
vec!["run", "-d", "--name", "box1", "alpine"]
200+
vec!["run", "--name", "box1", "alpine"]
203201
);
204202
}
205203

@@ -216,8 +214,20 @@ mod tests {
216214
assert_eq!(
217215
v,
218216
vec![
219-
"run", "-d", "--name", "box1", "--cpus", "2", "--memory", "256m", "-e", "A=1",
220-
"redis:7", "--", "redis-server", "--port", "0"
217+
"run",
218+
"--name",
219+
"box1",
220+
"--cpus",
221+
"2",
222+
"--memory",
223+
"256m",
224+
"-e",
225+
"A=1",
226+
"redis:7",
227+
"--",
228+
"redis-server",
229+
"--port",
230+
"0"
221231
]
222232
);
223233
}
@@ -247,23 +257,6 @@ mod tests {
247257
assert!(!is_running(""));
248258
}
249259

250-
#[test]
251-
fn status_name_maps() {
252-
assert_eq!(status_name(1), "created");
253-
assert_eq!(status_name(2), "running");
254-
assert_eq!(status_name(3), "stopped");
255-
assert_eq!(status_name(0), "unknown");
256-
assert_eq!(status_name(99), "unknown");
257-
}
258-
259-
#[test]
260-
fn parse_wait_exit_code_handles_garbage() {
261-
assert_eq!(parse_wait_exit_code("0\n"), 0);
262-
assert_eq!(parse_wait_exit_code(" 137 "), 137);
263-
assert_eq!(parse_wait_exit_code("notanumber"), 0);
264-
assert_eq!(parse_wait_exit_code(""), 0);
265-
}
266-
267260
#[test]
268261
fn a3s_home_default() {
269262
// Default applies when unset (don't mutate global env in parallel tests).

containerd-shim/src/main.rs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,17 @@ use service::Service;
2121

2222
#[tokio::main]
2323
async fn main() {
24-
containerd_shim::asynchronous::run::<Service>("io.containerd.a3s-box.v2", None).await;
24+
// Pass the Config via `opts` (NOT from Shim::new — the framework reads it to
25+
// set up signals + subreaper BEFORE calling Shim::new, so changes there are
26+
// no-ops). Disable the framework's child reaper and subreaper: it sets
27+
// PR_SET_CHILD_SUBREAPER and a SIGCHLD `waitpid(-1)` loop that reaps the
28+
// short-lived `a3s-box run -d` CLI and publishes a spurious container-exit
29+
// seconds after start, making containerd kill the still-running box. We hold
30+
// each task's process as a tokio Child and reap it ourselves in Task.wait().
31+
let config = containerd_shim::Config {
32+
no_reaper: true,
33+
no_sub_reaper: true,
34+
..Default::default()
35+
};
36+
containerd_shim::asynchronous::run::<Service>("io.containerd.a3s-box.v2", Some(config)).await;
2537
}

0 commit comments

Comments
 (0)