diff --git a/.github/scripts/native-linux-smoke.sh b/.github/scripts/native-linux-smoke.sh index 449789d..58e2a1e 100644 --- a/.github/scripts/native-linux-smoke.sh +++ b/.github/scripts/native-linux-smoke.sh @@ -43,7 +43,33 @@ run_smoke() { printf '%s\n' "$output" jq --exit-status \ --argjson expected "$expected_kvm_present" \ - '.status == "available" and .kvm_device_present == $expected' \ + '.schema_version == "a3s.oci.native-linux-smoke.v2" + and .platform == "linux" and .status == "available" + and .kvm_device_present == $expected + and .bundle_loaded + and .service_operations + == ["features", "create", "state", "start", "kill", "delete", "wait"] + and .dedicated_vm_rejected_before_create + and .create_returned_created + and .create_replayed + and (.created_pid > 0) + and .marker_absent_after_create + and .start_released + and .running_observed + and .kill_delivered + and .kill_replayed + and .wait_timeout_enforced + and .wait_exit_status == {"signal": 9, "oom_killed": false} + and .wait_replayed + and .stopped_observed + and .marker_verified + and .delete_succeeded + and .delete_replayed + and .state_missing_after_delete + and .marker_removed + and .executor_runtime_clean + and .session_root_clean + and (.reason == null)' \ <<<"$output" >/dev/null } @@ -59,12 +85,12 @@ run_multi_container_smoke() { printf '%s\n' "$output" jq --exit-status \ --argjson expected "$expected_kvm_present" \ - '.schema_version == "a3s.oci.native-linux-multi-container-smoke.v1" + '.schema_version == "a3s.oci.native-linux-multi-container-smoke.v2" and .platform == "linux" and .status == "available" and .kvm_device_present == $expected and .bundles_loaded and .service_operations - == ["features", "create", "state", "start", "kill", "delete"] + == ["features", "create", "state", "start", "kill", "delete", "wait"] and .lifecycle.distinct_bundle_directories and .lifecycle.distinct_rootfs_directories and .lifecycle.both_created_before_start @@ -80,8 +106,11 @@ run_multi_container_smoke() { and .lifecycle.marker_a_verified and .lifecycle.b_unchanged_after_a_start and .lifecycle.marker_b_absent_after_a_start + and .lifecycle.wait_a_did_not_block_b and .lifecycle.kill_a_replayed and .lifecycle.a_stopped + and .lifecycle.wait_status_a == {"signal": 9, "oom_killed": false} + and .lifecycle.wait_a_replayed and .lifecycle.b_unchanged_after_a_kill and .lifecycle.marker_b_absent_after_a_kill and .lifecycle.delete_a_replayed @@ -98,6 +127,8 @@ run_multi_container_smoke() { and .lifecycle.marker_b_verified and .lifecycle.kill_b_replayed and .lifecycle.b_stopped + and .lifecycle.wait_status_b == {"signal": 9, "oom_killed": false} + and .lifecycle.wait_b_replayed and .lifecycle.delete_b_replayed and .lifecycle.b_missing_after_delete and .markers_removed @@ -121,11 +152,11 @@ run_fault_cleanup() { --fault-after "$phase")" printf '%s\n' "$output" jq --exit-status --arg phase "$phase" \ - '.schema_version == "a3s.oci.native-linux-fault-cleanup.v1" + '.schema_version == "a3s.oci.native-linux-fault-cleanup.v2" and .platform == "linux" and .status == "available" and .bundle_loaded and .service_operations - == ["features", "create", "state", "start", "kill", "delete"] + == ["features", "create", "state", "start", "kill", "delete", "wait"] and .lifecycle.requested_fault == $phase and .lifecycle.injected_fault == $phase and .lifecycle.create_completed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3834824..3d16964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,7 +226,7 @@ jobs: if [[ "$support" == "1" ]]; then test "$status" -eq 0 jq --exit-status --arg agent_version "$agent_version" \ - '.schema_version == "a3s.oci.agent-vm-smoke.v3" + '.schema_version == "a3s.oci.agent-vm-smoke.v4" and .platform == "macos" and .status == "available" and .endpoint_bound and .shim_spawned and (.endpoint_name | startswith("a3s-oci-agent-")) @@ -234,11 +234,11 @@ jobs: and (.bridge_process_id > 0) and (.bridge_process_id != .shim_process_id) and .shim_client_verified and .protocol_negotiated - and .selected_protocol == 1 + and .selected_protocol == 2 and .agent_version == $agent_version and .guest_architecture == "aarch64" and .advertised_operations - == ["create", "state", "start", "kill", "delete"] + == ["create", "state", "start", "kill", "delete", "wait"] and .shim_report_verified and .shim_exit_code == 0 and .console_created and .shim_report.platform == "macos" @@ -256,7 +256,7 @@ jobs: else test "$status" -eq 2 jq --exit-status \ - '.schema_version == "a3s.oci.agent-vm-smoke.v3" + '.schema_version == "a3s.oci.agent-vm-smoke.v4" and .platform == "macos" and .status == "unavailable" and (.endpoint_name | startswith("a3s-oci-agent-")) and (.protocol_negotiated | not) @@ -287,7 +287,7 @@ jobs: printf '%s\n' "$unsigned_output" test "$unsigned_status" -eq 2 jq --exit-status \ - '.schema_version == "a3s.oci.agent-vm-smoke.v3" + '.schema_version == "a3s.oci.agent-vm-smoke.v4" and .platform == "macos" and .status == "unavailable" and .endpoint_bound and .shim_spawned and (.endpoint_name | startswith("a3s-oci-agent-")) @@ -346,7 +346,7 @@ jobs: if [[ "$support" == "1" ]]; then test "$status" -eq 0 jq --exit-status \ - '.schema_version == "a3s.oci.oci-vm-smoke.v3" + '.schema_version == "a3s.oci.oci-vm-smoke.v4" and .platform == "macos" and .status == "available" and .bundle_loaded and .create_returned_created and .create_replayed @@ -354,6 +354,10 @@ jobs: and .marker_absent_after_create and .start_released and .running_observed and .kill_delivered and .kill_replayed + and .wait_timeout_enforced + and .wait_exit_status + == {"exit_code": 0, "oom_killed": false} + and .wait_replayed and .stopped_observed and .marker_verified and .delete_succeeded and .delete_replayed and .state_missing_after_delete @@ -369,10 +373,10 @@ jobs: != .bridge.shim_process_id) and .bridge.shim_client_verified and .bridge.protocol_negotiated - and .bridge.selected_protocol == 1 + and .bridge.selected_protocol == 2 and .bridge.guest_architecture == "aarch64" and .bridge.advertised_operations - == ["create", "state", "start", "kill", "delete"] + == ["create", "state", "start", "kill", "delete", "wait"] and .bridge.shim_report_verified and .bridge.shim_exit_code == 0 and .bridge.console_created @@ -388,7 +392,7 @@ jobs: else test "$status" -eq 2 jq --exit-status \ - '.schema_version == "a3s.oci.oci-vm-smoke.v3" + '.schema_version == "a3s.oci.oci-vm-smoke.v4" and .platform == "macos" and .status == "unavailable" and .bundle_loaded and .bridge.platform == "macos" @@ -421,7 +425,7 @@ jobs: printf '%s\n' "$unsigned_output" test "$unsigned_status" -eq 2 jq --exit-status \ - '.schema_version == "a3s.oci.oci-vm-smoke.v3" + '.schema_version == "a3s.oci.oci-vm-smoke.v4" and .platform == "macos" and .status == "unavailable" and .bundle_loaded and (.create_returned_created | not) @@ -490,7 +494,7 @@ jobs: if [[ "$support" == "1" ]]; then test "$status" -eq 0 jq --exit-status \ - '.schema_version == "a3s.oci.oci-vm-multi-container-smoke.v1" + '.schema_version == "a3s.oci.oci-vm-multi-container-smoke.v2" and .platform == "macos" and .status == "available" and .bundles_loaded and .lifecycle.distinct_bundle_directories @@ -508,8 +512,12 @@ jobs: and .lifecycle.marker_a_verified and .lifecycle.b_unchanged_after_a_start and .lifecycle.marker_b_absent_after_a_start + and .lifecycle.wait_a_did_not_block_b and .lifecycle.kill_a_replayed and .lifecycle.a_stopped + and .lifecycle.wait_status_a + == {"exit_code": 0, "oom_killed": false} + and .lifecycle.wait_a_replayed and .lifecycle.b_unchanged_after_a_kill and .lifecycle.marker_b_absent_after_a_kill and .lifecycle.delete_a_replayed @@ -526,14 +534,18 @@ jobs: and .lifecycle.marker_b_verified and .lifecycle.kill_b_replayed and .lifecycle.b_stopped + and .lifecycle.wait_status_b + == {"exit_code": 0, "oom_killed": false} + and .lifecycle.wait_b_replayed and .lifecycle.delete_b_replayed and .lifecycle.b_missing_after_delete and .markers_removed and .guest_runtime_clean and .bridge.platform == "macos" and .bridge.status == "available" and .bridge.protocol_negotiated + and .bridge.selected_protocol == 2 and .bridge.advertised_operations - == ["create", "state", "start", "kill", "delete"] + == ["create", "state", "start", "kill", "delete", "wait"] and .bridge.shim_report_verified and .bridge.shim_exit_code == 0 and .bridge.macos_cleanup.endpoint_removed @@ -549,7 +561,7 @@ jobs: else test "$status" -eq 2 jq --exit-status \ - '.schema_version == "a3s.oci.oci-vm-multi-container-smoke.v1" + '.schema_version == "a3s.oci.oci-vm-multi-container-smoke.v2" and .platform == "macos" and .status == "unavailable" and .bundles_loaded and .bridge.platform == "macos" @@ -615,7 +627,7 @@ jobs: if [[ "$support" == "1" ]]; then test "$status" -eq 0 jq --exit-status --arg phase "$phase" \ - '.schema_version == "a3s.oci.oci-vm-fault-cleanup.v1" + '.schema_version == "a3s.oci.oci-vm-fault-cleanup.v2" and .platform == "macos" and .status == "available" and .bundle_loaded and .lifecycle.requested_fault == $phase @@ -640,6 +652,9 @@ jobs: and .marker_removed and .guest_runtime_clean and .bridge.status == "available" and .bridge.protocol_negotiated + and .bridge.selected_protocol == 2 + and .bridge.advertised_operations + == ["create", "state", "start", "kill", "delete", "wait"] and .bridge.shim_report_verified and .bridge.shim_exit_code == 0 and .bridge.macos_cleanup.endpoint_removed @@ -655,7 +670,7 @@ jobs: else test "$status" -eq 2 jq --exit-status --arg phase "$phase" \ - '.schema_version == "a3s.oci.oci-vm-fault-cleanup.v1" + '.schema_version == "a3s.oci.oci-vm-fault-cleanup.v2" and .platform == "macos" and .status == "unavailable" and .bundle_loaded and .lifecycle.requested_fault == $phase diff --git a/README.md b/README.md index 56d5c2f..77c3b78 100644 --- a/README.md +++ b/README.md @@ -80,13 +80,13 @@ Workload calls require an explicitly supplied launch-ready `RuntimeDriver`. - **Strict Validation**: Validate OCI 1.0.0 through 1.3.0 schemas, semantic relationships, paths, payload bounds, and immutable SHA-256 digests before state mutation -- **Durable Lifecycle**: Persist create, state, start, kill, and delete with - monotonic generations, operation IDs, replay, fencing, reconciliation, and - quarantine, backed by exhaustive durable-write and driver-boundary fault - matrices +- **Durable Lifecycle**: Persist create, start, kill, and delete with monotonic + generations, operation IDs, replay, fencing, reconciliation, and quarantine; + expose driver-backed state and stable init-process exit status - **Shared Linux Executor**: Reuse one fail-closed namespace, mount, pidfd - process-control, and cleanup implementation directly on Linux and through - the guest agent, with independently fenced per-container generations + process-control, exit-status, and cleanup implementation directly on Linux + and through the guest agent, with independently fenced per-container + generations - **Cross-Platform Drivers**: Inspect native Linux, KVM, HVF, and WHPX prerequisites without silently weakening requested isolation - **Typed SDK and IPC**: Expose an async `Send + Sync` Rust contract with @@ -138,7 +138,7 @@ prerequisite while remaining `probe-only`. ### Native Linux lifecycle The explicit rootful development driver proves the current OCI -create/start/kill/delete vertical slice without opening `/dev/kvm` or +create/start/kill/wait/delete vertical slice without opening `/dev/kvm` or initializing libkrun: ```sh @@ -159,7 +159,8 @@ sudo target/debug/a3s-oci native-linux-smoke \ --work-parent "$work_parent" ``` -Success requires the exact create/start barrier, running and stopped +Success requires the exact create/start barrier, a bounded wait while running, +the exact SIGKILL terminal result and repeated wait, running and stopped observation, idempotent mutation replay, marker verification, post-delete `NotFound`, and scoped cleanup. See [Native Linux Development](docs/linux-native.md) for the accepted profile and @@ -183,8 +184,10 @@ sudo target/debug/a3s-oci native-linux-multi-container-smoke \ The versioned report retains two simultaneous create barriers, distinct positive PIDs, operation replay isolation, A/B lifecycle independence, -generation-1 rejection after A is recreated as generation 2, and complete -process, marker, executor-root, and durable-session cleanup. +nonblocking observation of B while A is being waited on, exact repeated exit +status for both containers, generation-1 rejection after A is recreated as +generation 2, and complete process, marker, executor-root, and durable-session +cleanup. The separate fault-cleanup diagnostic deliberately stops before OCI delete and requires executor shutdown to reclaim the live process and all scoped state: @@ -199,7 +202,7 @@ for fault in after-create after-start after-kill; do done ``` -Each `a3s.oci.native-linux-fault-cleanup.v1` success identifies the exact +Each `a3s.oci.native-linux-fault-cleanup.v2` success identifies the exact injected boundary, records that normal delete was not attempted, and proves that the init PID, executor root, marker, and complete diagnostic session were removed. @@ -294,10 +297,11 @@ target/debug/a3s-oci agent-vm-smoke \ --console "$rootfs_dir/agent-console.log" ``` -An `a3s.oci.agent-vm-smoke.v3` success proves a private host socket, the +An `a3s.oci.agent-vm-smoke.v4` success proves a private host socket, the expected shim and direct worker PID relationship, one-time token -authentication, protocol version 1, the arm64 guest identity, and the exact -five core lifecycle operations. It also requires the exact runtime-owned +authentication, protocol version 2, the arm64 guest identity, and the exact +six guest operations (`create`, `state`, `start`, `kill`, `delete`, and +`wait`). It also requires the exact runtime-owned endpoint to be removed, the current process's complete descriptor inventory to return to its pre-session baseline, and both observed host process IDs to disappear. @@ -318,10 +322,12 @@ target/debug/a3s-oci oci-vm-smoke \ --console "$rootfs_dir/oci-console.log" ``` -Success proves distinct create and start, exact create/kill/delete replay, -running and stopped observation, marker verification, post-delete `NotFound`, -and nominal endpoint, process, marker, and guest-runtime cleanup. This remains -a fixed development profile rather than an arbitrary-workload driver. +Success proves distinct create and start, exact create/kill/delete replay, a +bounded wait while running, exact and replayed normal exit status after the +SIGTERM trap, running and stopped observation, marker verification, +post-delete `NotFound`, and nominal endpoint, process, marker, and +guest-runtime cleanup. This remains a fixed development profile rather than an +arbitrary-workload driver. The multi-container gate places two distinct bundles in the same guest and keeps both init processes behind the create barrier: @@ -340,11 +346,12 @@ target/debug/a3s-oci oci-vm-multi-container-smoke \ --console "$rootfs_dir/oci-multi-container.log" ``` -`a3s.oci.oci-vm-multi-container-smoke.v1` requires that starting, killing, and -deleting A never changes B; recreating A advances generation 1 to 2; stale and -cross-container replay requests fail; B then completes independently; and VM -shutdown restores guest-runtime, endpoint, descriptor, shim, and worker -inventories. +`a3s.oci.oci-vm-multi-container-smoke.v2` requires that starting, killing, +waiting for, and deleting A never changes or blocks B; both waits return and +replay the exact normal exit status; recreating A advances generation 1 to 2; +stale and cross-container replay requests fail; B then completes +independently; and VM shutdown restores guest-runtime, endpoint, descriptor, +shim, and worker inventories. Fault cleanup reuses the same signed shim and bundle but stops after each successful lifecycle boundary: @@ -361,7 +368,7 @@ for fault in after-create after-start after-kill; do done ``` -An `a3s.oci.oci-vm-fault-cleanup.v1` success requires no normal delete call, +An `a3s.oci.oci-vm-fault-cleanup.v2` success requires no normal delete call, guest executor shutdown, marker and runtime-root removal, exact endpoint removal, shim and VM-worker reap, and restoration of the complete host descriptor inventory. @@ -424,7 +431,8 @@ The current executor implements a reviewed bootstrap vertical slice: - ordered existing-target OCI mounts with bind/rbind and common VFS options; - PID-authenticated create/start barrier; - credentials, umask, `no_new_privileges`, `execve`, PID-reuse-safe pidfd - signaling, observation, and scoped cleanup. + signaling, exact normal-or-signal exit status, repeated wait, observation, + and scoped cleanup. Unimplemented OCI fields are rejected instead of ignored. User and time namespaces, namespace joins, complete mount semantics, cgroup resources, @@ -441,9 +449,12 @@ operations are still release gates. - checkpoint and restore; - typed IDs, operation IDs, deadlines, generations, and isolation requests. -The durable host currently implements the five core lifecycle mutations around -an injected `RuntimeDriver`. Methods without enforcement remain explicitly -unsupported and are not advertised early. +The durable host implements the five core lifecycle operations around an +injected `RuntimeDriver` and exposes `wait` only when that exact driver +implements it. The native Linux driver and protocol-v2 Linux guest executor +return a stable normal-exit or signal result across repeated waits. Methods +without enforcement remain explicitly unsupported and are not advertised +early. The local IPC and guest-agent protocols are versioned, length-delimited, and 64 MiB bounded. Every untrusted request is revalidated at the receiving @@ -453,9 +464,9 @@ boundary. | Host | Execution path | Retained evidence | Current readiness | | --- | --- | --- | --- | -| Linux x86_64/aarch64 | Native Linux executor | Kernel pidfd signaling probe, real rootful core lifecycle, two-container generation and mutation isolation, plus shutdown cleanup after create, start, and kill without delete; `/dev/kvm` absent and present-but-unusable | Default inventory `probe-only`; explicitly opened development instance `experimental` | +| Linux x86_64/aarch64 | Native Linux executor | Kernel pidfd signaling probe, real rootful lifecycle with exact SIGKILL status and repeated wait, two-container wait and mutation isolation, plus shutdown cleanup after create, start, and kill without delete; `/dev/kvm` absent and present-but-unusable | Default inventory `probe-only`; explicitly opened development instance `experimental` | | Linux x86_64/aarch64 | libkrun + KVM utility VM | Device access, ioctl result, and KVM API version | `probe-only`; VM driver not implemented | -| macOS arm64 | libkrun + HVF utility VM | Direct HVF VM create/destroy, checksum-pinned context lifecycle, authenticated static arm64 guest agent, pidfd-backed fixed and two-container OCI lifecycles, and no-delete cleanup after create, start, and kill | `probe-only`; complete enforcement and recovery pending | +| macOS arm64 | libkrun + HVF utility VM | Direct HVF VM create/destroy, checksum-pinned context lifecycle, authenticated protocol-v2 arm64 guest agent, pidfd-backed fixed and two-container OCI lifecycles with exact repeated exit status and nonblocking wait evidence, and no-delete cleanup after create, start, and kill | `probe-only`; complete enforcement and recovery pending | | Windows x86_64 | libkrun + WHPX utility VM | Partition, context, guest command, authenticated agent, and fixed OCI core lifecycle | `probe-only`; complete enforcement and recovery pending | Linux installation, feature inspection, and the native SDK path must work when @@ -580,7 +591,7 @@ cargo clippy \ Platform CI covers: -- the 237-point durable commit matrix and all 12 `RuntimeDriver` call +- the 237-point durable commit matrix and all 14 `RuntimeDriver` call boundaries on Linux, macOS, and Windows; - Ubuntu x86_64 native pidfd probe, lifecycle, multi-container isolation, and three-phase no-delete cleanup without KVM; diff --git a/ROADMAP.md b/ROADMAP.md index fefbf4f..b1bd0f4 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -53,21 +53,23 @@ Completed: host-visible marker verification, bounded worker reap, and marker cleanup; - real macOS static arm64 guest-agent boot through AF_VSOCK and a private Unix socket, with `LOCAL_PEERPID`, direct shim-worker parent verification, - one-time token authentication, protocol-v1 negotiation, exact core - operation advertisement, process-group termination, exact endpoint removal, + one-time token authentication, protocol-v2 negotiation, exact six-operation + advertisement, process-group termination, exact endpoint removal, observed PID reap, and in-process descriptor-inventory restoration; -- real macOS fixed-bundle create/state/start/kill/delete evidence using the - shared Windows lifecycle harness, including exact mutation retries, - create/start separation, running and stopped observation, post-delete - NotFound, and nominal process, endpoint, marker, and runtime-root cleanup; +- real macOS fixed-bundle create/state/start/kill/wait/delete evidence using + the shared Windows lifecycle harness, including exact mutation retries, + create/start separation, bounded running wait, exact repeated normal exit + status, running and stopped observation, post-delete NotFound, and nominal + process, endpoint, marker, and runtime-root cleanup; - real macOS no-delete cleanup after successful create, start, and kill boundaries, with exact fault identity, guest executor shutdown, endpoint and marker removal, shim/worker reap, descriptor-inventory restoration, and no new guest runtime root; - explicit rootful native Linux driver integration that reuses the shared executor without linking or initializing libkrun; -- real native Linux create/state/start/kill/delete SDK evidence on x86_64 and - aarch64, repeated with `/dev/kvm` absent and present but unusable; +- real native Linux create/state/start/kill/wait/delete SDK evidence on x86_64 + and aarch64, including exact repeated SIGKILL status and bounded running + wait, repeated with `/dev/kvm` absent and present but unusable; - real native Linux no-delete cleanup after create, start, and kill on x86_64 and aarch64, including init-PID reap and executor, durable-state, marker, and session-root removal; @@ -107,16 +109,17 @@ Completed: - version-negotiated, length-delimited transport for every SDK operation; - tested Windows named-pipe and Unix-domain-socket client connectors; - authenticated, version-negotiated, bounded host/guest lifecycle protocol - with exact bundle and response correlation; + with exact bundle and response correlation, protocol-v1 compatibility, and + protocol-v2 stable init wait; - existing `features` CLI path routed through the Rust SDK; - single-writer durable state for the complete core lifecycle, with exact bundle snapshots, monotonic generations, generation fencing, global idempotent create/start/kill/delete journals, active-operation claims, terminal failure replay, crash reconciliation, and quarantine; - async `RuntimeDriver` integration plus a tested host implementation of - `create`, `state`, `start`, `kill`, and `delete`; + `create`, `state`, `start`, `kill`, `delete`, and driver-advertised `wait`; - typed, exhaustive recovery injection at all 237 registered durable commit - stages and all 12 before/after `RuntimeDriver` method boundaries; + stages and all 14 before/after `RuntimeDriver` method boundaries; - runtime-owned Windows state paths with protected DACLs limited to the runtime principal and LocalSystem, inheritance disabled, and every applied owner and ACL verified; @@ -136,7 +139,8 @@ Not yet complete: The built-in WHPX driver remains `probe-only`, and the default host service advertises only `features`. A host explicitly opened around a launch-ready -`RuntimeDriver` advertises the five durable core lifecycle operations. +`RuntimeDriver` advertises the five required core lifecycle operations plus +only the optional operations that driver implements. ## Delivery Sequence @@ -251,9 +255,9 @@ runtime-root leak. Only then may WHPX become `experimental`. - [ ] Boot the pinned A3S Linux kernel and immutable system root. - [x] Establish the private macOS Unix endpoint and AF_VSOCK guest-agent bridge, verify that the peer is the shim's direct VM worker child, and - authenticate protocol-v1 negotiation with a one-time token. -- [x] Run the same fixed create/state/start/kill/delete OCI lifecycle used by - WHPX. + authenticate protocol-v2 negotiation with a one-time token. +- [x] Run the same fixed create/state/start/kill/wait/delete OCI lifecycle used + by WHPX, including bounded running wait and exact repeated exit status. - [x] Prove deterministic VM, process, descriptor, and filesystem cleanup without normal delete after successful create, start, and kill boundaries. Each phase requires exact endpoint removal, observed-PID reap, complete @@ -272,9 +276,9 @@ then may HVF become `experimental`. - [x] Multi-container guest registry with per-container generations, proven with two distinct bundles, simultaneous create barriers, independent - start/kill/delete, exact replay isolation, generation-1 fencing after - generation-2 recreation, and complete cleanup through native Linux and the - macOS utility VM. + start/kill/wait/delete, nonblocking wait/state progress, exact replay + isolation, generation-1 fencing after generation-2 recreation, and complete + cleanup through native Linux and the macOS utility VM. - [x] Create a new UTS namespace and apply the configured hostname and domainname before the created barrier. - [x] Create a new mount namespace, make the inherited mount tree recursively @@ -293,6 +297,9 @@ then may HVF become `experimental`. kernels without `pidfd_open` and `pidfd_send_signal`, and deliver lifecycle and cleanup signals without a numeric-PID reuse race. Prove the path through native Linux and the macOS utility VM. +- [x] Retain exact normal-or-signal init termination, return the same result + from repeated waits, enforce bounded wait timeouts, and prove one container's + wait does not block another container's state request. - [ ] Namespace creation for user and time namespaces, plus joining existing namespaces. - [ ] Mount-target creation, rootfs propagation overrides, idmapped and @@ -302,7 +309,7 @@ then may HVF become `experimental`. priority, affinity, `no_new_privileges`, LSMs, and seccomp. - [ ] cgroup v2 CPU, memory, pids, I/O, hugepage, RDMA, device, and unified resource enforcement. -- [ ] Init supervision, zombie reaping, exec, and wait. +- [ ] Namespace-internal init supervision, orphan/zombie reaping, and exec. - [ ] Ordered hooks with OCI state on stdin. - [ ] Backpressured stdin/stdout/stderr, PTY, resize, signals, and output cursors. @@ -320,7 +327,8 @@ and recovery suites in the Windows guest and on native Linux. - [x] Add the native Linux driver without linking or initializing libkrun. - [x] Reuse the R3 Linux executor directly. - [x] Prove runtime binary startup, feature inspection, Rust SDK loading, and - the rootful core lifecycle without KVM on x86_64 and aarch64. + the rootful lifecycle including exact repeated init wait without KVM on + x86_64 and aarch64. - [x] Prove shutdown cleanup without delete after create, start, and kill on x86_64 and aarch64 without KVM. - [ ] Prove packaged installation and A3S Box product startup without KVM. diff --git a/crates/agent-protocol/src/client.rs b/crates/agent-protocol/src/client.rs index 5a6b0fd..e0c87ce 100644 --- a/crates/agent-protocol/src/client.rs +++ b/crates/agent-protocol/src/client.rs @@ -1,19 +1,22 @@ use std::fmt; use std::sync::Arc; +use std::time::{Duration, Instant}; use a3s_oci_sdk::oci_spec::runtime::ContainerState; -use a3s_oci_sdk::{Error, ErrorCode, Result}; +use a3s_oci_sdk::{Error, ErrorCode, ExitStatus, Result}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::sync::Mutex; use crate::model::{ protocol_error, AgentCreateRequest, AgentDeleteRequest, AgentHello, AgentKillRequest, AgentOperation, AgentRequest, AgentResponse, AgentStartRequest, AgentState, AgentStateRequest, - HelloOutcome, HostHello, ProtocolRange, RequestEnvelope, ResponseEnvelope, ResponseOutcome, - SessionToken, + AgentWaitRequest, HelloOutcome, HostHello, ProtocolRange, RequestEnvelope, ResponseEnvelope, + ResponseOutcome, SessionToken, }; use crate::wire::{read_frame, write_frame}; +const WAIT_POLL_INTERVAL: Duration = Duration::from_millis(25); + /// Authenticated, correlated client for one guest-agent stream. pub struct AgentClient { connection: Arc>>, @@ -116,13 +119,52 @@ where pub async fn delete(&self, request: AgentDeleteRequest) -> Result<()> { match self.call(AgentRequest::Delete(request)).await? { AgentResponse::Deleted => Ok(()), - AgentResponse::State(_) => Err(protocol_error( + AgentResponse::State(_) | AgentResponse::ExitStatus(_) => Err(protocol_error( ErrorCode::Internal, - "guest returned state for an OCI delete request", + "guest returned the wrong response for an OCI delete request", )), } } + /// Wait for one exact init process and return its stable terminal result. + pub async fn wait(&self, request: AgentWaitRequest) -> Result { + let limit = request.timeout_ms.map(Duration::from_millis); + let started = Instant::now(); + loop { + let poll_timeout = match limit { + Some(limit) => WAIT_POLL_INTERVAL.min(limit.saturating_sub(started.elapsed())), + None => WAIT_POLL_INTERVAL, + }; + let poll = AgentWaitRequest { + target: request.target.clone(), + timeout_ms: Some(duration_millis(poll_timeout)), + }; + match self.call(AgentRequest::Wait(poll)).await { + Ok(AgentResponse::ExitStatus(status)) => return Ok(status), + Ok(AgentResponse::State(_) | AgentResponse::Deleted) => { + return Err(protocol_error( + ErrorCode::Internal, + "guest returned the wrong response for an OCI wait request", + )); + } + Err(error) if error.code == ErrorCode::DeadlineExceeded => { + if limit.is_some_and(|limit| started.elapsed() >= limit) { + return Err(protocol_error( + ErrorCode::DeadlineExceeded, + format!( + "timed out after {} ms waiting for container {}", + request.timeout_ms.unwrap_or_default(), + request.target.id + ), + ) + .retryable(true)); + } + } + Err(error) => return Err(error), + } + } + } + async fn call(&self, request: AgentRequest) -> Result { request.validate()?; ensure_advertised(self.hello.capabilities().operations(), &request)?; @@ -187,6 +229,10 @@ where } } +fn duration_millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + fn ensure_advertised(operations: &[AgentOperation], request: &AgentRequest) -> Result<()> { let required = match request { AgentRequest::Create(_) => AgentOperation::Create, @@ -194,6 +240,7 @@ fn ensure_advertised(operations: &[AgentOperation], request: &AgentRequest) -> R AgentRequest::Start(_) => AgentOperation::Start, AgentRequest::Kill(_) => AgentOperation::Kill, AgentRequest::Delete(_) => AgentOperation::Delete, + AgentRequest::Wait(_) => AgentOperation::Wait, }; if operations.contains(&required) { Ok(()) @@ -237,6 +284,7 @@ fn validate_response_for_request(request: &AgentRequest, response: &AgentRespons validate_state_target(&request.target, state) } (AgentRequest::Delete(_), AgentResponse::Deleted) => Ok(()), + (AgentRequest::Wait(_), AgentResponse::ExitStatus(status)) => status.validate(), (request, response) => Err(protocol_error( ErrorCode::Internal, format!( @@ -267,7 +315,7 @@ fn validate_state_target( fn expect_state(response: AgentResponse, operation: &'static str) -> Result { match response { AgentResponse::State(state) => Ok(state), - AgentResponse::Deleted => Err(protocol_error( + AgentResponse::Deleted | AgentResponse::ExitStatus(_) => Err(protocol_error( ErrorCode::Internal, format!("guest returned delete acknowledgement for OCI {operation}"), )), @@ -295,6 +343,7 @@ const fn request_name(request: &AgentRequest) -> &'static str { AgentRequest::Start(_) => "start", AgentRequest::Kill(_) => "kill", AgentRequest::Delete(_) => "delete", + AgentRequest::Wait(_) => "wait", } } diff --git a/crates/agent-protocol/src/lib.rs b/crates/agent-protocol/src/lib.rs index 97e5047..2d5ef23 100644 --- a/crates/agent-protocol/src/lib.rs +++ b/crates/agent-protocol/src/lib.rs @@ -15,8 +15,9 @@ pub use client::AgentClient; pub use model::{ AgentBundle, AgentCapabilities, AgentCreateRequest, AgentDeleteRequest, AgentHello, AgentKillRequest, AgentOperation, AgentRequest, AgentResponse, AgentStartRequest, AgentState, - AgentStateRequest, GuestPath, SessionToken, AGENT_MAX_FRAME_BYTES, AGENT_PROTOCOL_VERSION_MAX, - AGENT_PROTOCOL_VERSION_MIN, AGENT_SESSION_TOKEN_BYTES, AGENT_SESSION_TOKEN_ENV, + AgentStateRequest, AgentWaitRequest, GuestPath, SessionToken, AGENT_MAX_FRAME_BYTES, + AGENT_PROTOCOL_VERSION_MAX, AGENT_PROTOCOL_VERSION_MIN, AGENT_SESSION_TOKEN_BYTES, + AGENT_SESSION_TOKEN_ENV, }; pub use server::{serve_agent_connection, GuestAgentService}; pub use transport::{AgentVsockEndpoint, AGENT_VSOCK_PORT}; diff --git a/crates/agent-protocol/src/model.rs b/crates/agent-protocol/src/model.rs index 14cbb96..002b0ec 100644 --- a/crates/agent-protocol/src/model.rs +++ b/crates/agent-protocol/src/model.rs @@ -3,8 +3,8 @@ use std::path::PathBuf; use a3s_oci_sdk::oci_spec::runtime::ContainerState; use a3s_oci_sdk::{ - ContainerTarget, DeleteMode, Error, ErrorCode, OciBundle, OperationContext, ProcessIo, Result, - Signal, + ContainerTarget, DeleteMode, Error, ErrorCode, ExitStatus, OciBundle, OperationContext, + ProcessIo, Result, Signal, }; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; @@ -12,7 +12,7 @@ use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; /// Oldest host-to-guest protocol version implemented by this build. pub const AGENT_PROTOCOL_VERSION_MIN: u16 = 1; /// Newest host-to-guest protocol version implemented by this build. -pub const AGENT_PROTOCOL_VERSION_MAX: u16 = 1; +pub const AGENT_PROTOCOL_VERSION_MAX: u16 = 2; /// Maximum encoded host-to-guest frame size. pub const AGENT_MAX_FRAME_BYTES: u32 = 64 * 1024 * 1024; /// Required session-token entropy supplied by the host. @@ -238,7 +238,7 @@ impl AgentBundle { } } -/// Guest operations available in protocol version 1. +/// Guest operations available in the negotiated protocol. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum AgentOperation { @@ -247,6 +247,8 @@ pub enum AgentOperation { Start, Kill, Delete, + /// Wait for init termination. Available from protocol version 2. + Wait, } /// Runtime properties reported by the guest during negotiation. @@ -286,7 +288,26 @@ impl AgentCapabilities { ) } - /// Construct an exact protocol-v1 capability report. + /// Construct the Linux executor capability report, including protocol-v2 wait. + pub fn linux_executor( + agent_version: impl Into, + architecture: impl Into, + ) -> Result { + Self::new( + agent_version, + architecture, + vec![ + AgentOperation::Create, + AgentOperation::State, + AgentOperation::Start, + AgentOperation::Kill, + AgentOperation::Delete, + AgentOperation::Wait, + ], + ) + } + + /// Construct an exact capability report. pub fn new( agent_version: impl Into, architecture: impl Into, @@ -340,7 +361,35 @@ impl AgentCapabilities { if self.max_frame_bytes != AGENT_MAX_FRAME_BYTES { return Err(protocol_error( ErrorCode::InvalidArgument, - format!("protocol v1 requires maxFrameBytes={AGENT_MAX_FRAME_BYTES}"), + format!("agent protocol requires maxFrameBytes={AGENT_MAX_FRAME_BYTES}"), + )); + } + Ok(()) + } + + pub(crate) fn for_protocol(&self, selected_version: u16) -> Result { + let mut capabilities = self.clone(); + if selected_version == 1 { + capabilities + .operations + .retain(|operation| *operation != AgentOperation::Wait); + } + capabilities.validate_for_protocol(selected_version)?; + Ok(capabilities) + } + + pub(crate) fn validate_for_protocol(&self, selected_version: u16) -> Result<()> { + self.validate()?; + if !(AGENT_PROTOCOL_VERSION_MIN..=AGENT_PROTOCOL_VERSION_MAX).contains(&selected_version) { + return Err(protocol_error( + ErrorCode::FailedPrecondition, + format!("unsupported agent protocol version {selected_version}"), + )); + } + if selected_version == 1 && self.operations.contains(&AgentOperation::Wait) { + return Err(protocol_error( + ErrorCode::FailedPrecondition, + "protocol version 1 cannot advertise wait", )); } Ok(()) @@ -398,6 +447,17 @@ pub struct AgentStateRequest { pub target: ContainerTarget, } +/// Wait for one exact container init process. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AgentWaitRequest { + /// Container ID plus a positive exact generation. + pub target: ContainerTarget, + /// Maximum wait duration. `None` waits without a protocol-imposed deadline. + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, +} + /// OCI start input sent to the guest executor. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -445,6 +505,7 @@ pub enum AgentRequest { Start(AgentStartRequest), Kill(AgentKillRequest), Delete(AgentDeleteRequest), + Wait(AgentWaitRequest), } /// Guest-observed init-process state for one exact generation. @@ -507,6 +568,7 @@ impl AgentState { pub enum AgentResponse { State(AgentState), Deleted, + ExitStatus(ExitStatus), } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/agent-protocol/src/server.rs b/crates/agent-protocol/src/server.rs index b1c9142..e61841d 100644 --- a/crates/agent-protocol/src/server.rs +++ b/crates/agent-protocol/src/server.rs @@ -1,13 +1,13 @@ use std::sync::Arc; -use a3s_oci_sdk::{async_trait, Error, ErrorCode, Result}; +use a3s_oci_sdk::{async_trait, Error, ErrorCode, ExitStatus, Result}; use tokio::io::{AsyncRead, AsyncWrite}; use crate::model::{ protocol_error, AgentCapabilities, AgentCreateRequest, AgentDeleteRequest, AgentHello, AgentKillRequest, AgentRequest, AgentResponse, AgentStartRequest, AgentState, - AgentStateRequest, HelloOutcome, HostHello, RequestEnvelope, ResponseEnvelope, ResponseOutcome, - SessionToken, + AgentStateRequest, AgentWaitRequest, HelloOutcome, HostHello, RequestEnvelope, + ResponseEnvelope, ResponseOutcome, SessionToken, }; use crate::validation::negotiate_protocol; use crate::wire::{read_frame, write_frame}; @@ -36,6 +36,11 @@ pub trait GuestAgentService: Send + Sync { /// Delete only resources owned by the requested generation. async fn delete(&self, request: AgentDeleteRequest) -> Result<()>; + + /// Wait for the exact container init process. + async fn wait(&self, _request: AgentWaitRequest) -> Result { + Err(Error::unsupported("agent-wait")) + } } /// Authenticate, negotiate, and serve one host connection until clean EOF. @@ -66,7 +71,19 @@ where return Err(error); } }; - let capabilities = service.capabilities(); + let capabilities = match service.capabilities().for_protocol(selected_version) { + Ok(capabilities) => capabilities, + Err(error) => { + write_frame( + &mut stream, + &HelloOutcome::Rejected { + error: error.clone(), + }, + ) + .await?; + return Err(error); + } + }; let hello = AgentHello::new(selected_version, capabilities); write_frame(&mut stream, &HelloOutcome::Accepted { hello }).await?; @@ -128,6 +145,7 @@ async fn dispatch(service: &dyn GuestAgentService, request: AgentRequest) -> Res service.delete(request).await?; Ok(AgentResponse::Deleted) } + AgentRequest::Wait(request) => service.wait(request).await.map(AgentResponse::ExitStatus), } } diff --git a/crates/agent-protocol/src/tests.rs b/crates/agent-protocol/src/tests.rs index a30fc45..f502bf5 100644 --- a/crates/agent-protocol/src/tests.rs +++ b/crates/agent-protocol/src/tests.rs @@ -1,17 +1,18 @@ use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use a3s_oci_sdk::oci_spec::runtime::ContainerState; use a3s_oci_sdk::{ - async_trait, ContainerId, ContainerTarget, DeleteMode, Error, ErrorCode, Generation, OciBundle, - OperationContext, OperationId, ProcessIo, Result, Signal, + async_trait, ContainerId, ContainerTarget, DeleteMode, Error, ErrorCode, ExitStatus, + Generation, OciBundle, OperationContext, OperationId, ProcessIo, Result, Signal, }; use tokio::io::{AsyncWriteExt, DuplexStream}; use crate::model::{ AgentCreateRequest, AgentDeleteRequest, AgentHello, AgentKillRequest, AgentRequest, - AgentResponse, AgentStartRequest, AgentState, AgentStateRequest, HelloOutcome, HostHello, - RequestEnvelope, ResponseEnvelope, ResponseOutcome, + AgentResponse, AgentStartRequest, AgentState, AgentStateRequest, AgentWaitRequest, + HelloOutcome, HostHello, ProtocolRange, RequestEnvelope, ResponseEnvelope, ResponseOutcome, }; use crate::wire::{read_frame, read_frame_for_test, write_frame}; use crate::{ @@ -35,19 +36,21 @@ const TEST_CONFIG: &str = concat!( #[derive(Debug, Default)] struct TestAgent { state: Mutex, + wait_dispatches: AtomicUsize, } #[derive(Debug, Default)] struct TestAgentState { states: HashMap, highest_generations: HashMap, + exits: HashMap, next_pid: i32, } #[async_trait] impl GuestAgentService for TestAgent { fn capabilities(&self) -> AgentCapabilities { - AgentCapabilities::core("0.1.0-test", std::env::consts::ARCH) + AgentCapabilities::linux_executor("0.1.0-test", std::env::consts::ARCH) .expect("valid test capabilities") } @@ -157,10 +160,33 @@ impl GuestAgentService for TestAgent { None, digest, )?; + agent.exits.insert( + request.target.id.clone(), + ExitStatus::signaled(request.signal.get(), false)?, + ); agent.states.insert(request.target.id, state.clone()); Ok(state) } + async fn wait(&self, request: AgentWaitRequest) -> Result { + self.wait_dispatches.fetch_add(1, Ordering::SeqCst); + let agent = self.state.lock().expect("agent state lock"); + let current = agent + .states + .get(&request.target.id) + .ok_or_else(|| Error::new(ErrorCode::NotFound, "guest container does not exist"))?; + if current.target() != &request.target { + return Err(Error::new( + ErrorCode::Conflict, + "guest container generation does not match", + )); + } + agent.exits.get(&request.target.id).cloned().ok_or_else(|| { + Error::new(ErrorCode::DeadlineExceeded, "test container has not exited") + .for_operation("agent-wait") + }) + } + async fn delete(&self, request: AgentDeleteRequest) -> Result<()> { let mut agent = self.state.lock().expect("agent state lock"); let current = agent @@ -216,18 +242,22 @@ fn spawn_server( stream: DuplexStream, expected_token: SessionToken, ) -> tokio::task::JoinHandle> { - tokio::spawn(serve_agent_connection( - stream, - expected_token, - Arc::new(TestAgent::default()), - )) + spawn_server_with_agent(stream, expected_token, Arc::new(TestAgent::default())) +} + +fn spawn_server_with_agent( + stream: DuplexStream, + expected_token: SessionToken, + agent: Arc, +) -> tokio::task::JoinHandle> { + tokio::spawn(serve_agent_connection(stream, expected_token, agent)) } #[tokio::test] async fn negotiates_and_round_trips_the_core_oci_lifecycle() { let (host, guest) = tokio::io::duplex(1024 * 1024); let server = spawn_server(guest, token(7)); - let client = AgentClient::connect(host, token(7)) + let client = AgentClient::connect_for_test(host, token(7), 1, 1) .await .expect("connect agent client"); assert_eq!(client.hello().selected_version(), 1); @@ -286,6 +316,134 @@ async fn negotiates_and_round_trips_the_core_oci_lifecycle() { .expect("clean server shutdown"); } +#[tokio::test] +async fn protocol_v2_wait_returns_and_replays_the_exact_exit_status() { + let (host, guest) = tokio::io::duplex(1024 * 1024); + let server = spawn_server(guest, token(18)); + let client = AgentClient::connect(host, token(18)) + .await + .expect("connect protocol-v2 client"); + assert_eq!(client.hello().selected_version(), 2); + assert_eq!( + client.hello().capabilities().operations(), + &[ + crate::AgentOperation::Create, + crate::AgentOperation::State, + crate::AgentOperation::Start, + crate::AgentOperation::Kill, + crate::AgentOperation::Delete, + crate::AgentOperation::Wait, + ] + ); + + let create = create_request_for("wait-container", 1, "wait-create"); + let target = create.target.clone(); + let digest = create.bundle.config_digest().to_string(); + client.create(create).await.expect("create wait container"); + client + .start(AgentStartRequest { + context: OperationContext::new(operation_id("wait-start")), + target: target.clone(), + expected_config_digest: digest, + }) + .await + .expect("start wait container"); + client + .kill(AgentKillRequest { + context: OperationContext::new(operation_id("wait-kill")), + target: target.clone(), + signal: Signal::new(9).expect("signal"), + all: false, + }) + .await + .expect("kill wait container"); + let request = AgentWaitRequest { + target: target.clone(), + timeout_ms: Some(1_000), + }; + let expected = ExitStatus::signaled(9, false).expect("exit status"); + assert_eq!( + client.wait(request.clone()).await.expect("first wait"), + expected + ); + assert_eq!(client.wait(request).await.expect("repeated wait"), expected); + + client + .delete(AgentDeleteRequest { + context: OperationContext::new(operation_id("wait-delete")), + target, + mode: DeleteMode::StoppedOnly, + }) + .await + .expect("delete wait container"); + drop(client); + server + .await + .expect("server task") + .expect("clean protocol-v2 server shutdown"); +} + +#[tokio::test] +async fn protocol_v1_rejects_a_forged_wait_before_service_dispatch() { + let (mut host, guest) = tokio::io::duplex(64 * 1024); + let agent = Arc::new(TestAgent::default()); + let expected_token = token(19); + let server = spawn_server_with_agent(guest, expected_token.clone(), agent.clone()); + + write_frame( + &mut host, + &HostHello { + protocols: ProtocolRange { min: 1, max: 1 }, + token: expected_token, + }, + ) + .await + .expect("write protocol-v1 hello"); + let hello: HelloOutcome = read_frame(&mut host) + .await + .expect("read protocol-v1 hello") + .expect("server returned protocol-v1 hello"); + let HelloOutcome::Accepted { hello } = hello else { + panic!("protocol-v1 negotiation was rejected"); + }; + assert_eq!(hello.selected_version(), 1); + assert!(!hello + .capabilities() + .operations() + .contains(&crate::AgentOperation::Wait)); + + write_frame( + &mut host, + &RequestEnvelope { + version: 1, + request_id: 41, + request: AgentRequest::Wait(AgentWaitRequest { + target: ContainerTarget::exact(container_id("forged-wait"), Generation(1)), + timeout_ms: Some(1), + }), + }, + ) + .await + .expect("write forged protocol-v1 wait"); + let response: ResponseEnvelope = read_frame(&mut host) + .await + .expect("read forged wait response") + .expect("server returned forged wait response"); + assert_eq!(response.version, 1); + assert_eq!(response.request_id, 41); + let ResponseOutcome::Failed { error } = response.outcome else { + panic!("forged protocol-v1 wait unexpectedly succeeded"); + }; + assert_eq!(error.code, ErrorCode::Unsupported); + assert_eq!(agent.wait_dispatches.load(Ordering::SeqCst), 0); + + drop(host); + server + .await + .expect("server task") + .expect("clean protocol-v1 server shutdown"); +} + #[tokio::test] async fn transports_two_independently_fenced_container_generations() { let (host, guest) = tokio::io::duplex(1024 * 1024); @@ -432,7 +590,7 @@ async fn rejects_wrong_session_tokens_and_incompatible_versions() { let (host, guest) = tokio::io::duplex(64 * 1024); let server = spawn_server(guest, token(9)); - let error = AgentClient::connect_for_test(host, token(9), 2, 2) + let error = AgentClient::connect_for_test(host, token(9), 3, 3) .await .expect_err("incompatible version must fail"); assert_eq!(error.code, ErrorCode::FailedPrecondition); diff --git a/crates/agent-protocol/src/validation.rs b/crates/agent-protocol/src/validation.rs index bf9d476..98b8824 100644 --- a/crates/agent-protocol/src/validation.rs +++ b/crates/agent-protocol/src/validation.rs @@ -8,8 +8,8 @@ use a3s_oci_sdk::{ use crate::model::{ protocol_error, AgentBundle, AgentCreateRequest, AgentDeleteRequest, AgentHello, AgentKillRequest, AgentRequest, AgentResponse, AgentStartRequest, AgentState, - AgentStateRequest, ProtocolRange, RequestEnvelope, ResponseEnvelope, ResponseOutcome, - AGENT_MAX_FRAME_BYTES, + AgentStateRequest, AgentWaitRequest, ProtocolRange, RequestEnvelope, ResponseEnvelope, + ResponseOutcome, AGENT_MAX_FRAME_BYTES, }; impl AgentBundle { @@ -50,6 +50,12 @@ impl AgentStateRequest { } } +impl AgentWaitRequest { + pub(crate) fn validate(&self) -> Result<()> { + validate_exact_target(&self.target) + } +} + impl AgentStartRequest { pub(crate) fn validate(&self) -> Result<()> { validate_exact_target(&self.target)?; @@ -77,6 +83,30 @@ impl AgentRequest { Self::Start(request) => request.validate(), Self::Kill(request) => request.validate(), Self::Delete(request) => request.validate(), + Self::Wait(request) => request.validate(), + } + } + + pub(crate) fn validate_for_protocol(&self, selected_version: u16) -> Result<()> { + self.validate()?; + if selected_version < self.minimum_protocol_version() { + return Err(protocol_error( + ErrorCode::Unsupported, + format!( + "agent request requires protocol version {}, negotiated {selected_version}", + self.minimum_protocol_version() + ), + )); + } + Ok(()) + } + + const fn minimum_protocol_version(&self) -> u16 { + match self { + Self::Create(_) | Self::State(_) | Self::Start(_) | Self::Kill(_) | Self::Delete(_) => { + 1 + } + Self::Wait(_) => 2, } } } @@ -101,8 +131,20 @@ impl AgentResponse { match self { Self::State(state) => state.validate(), Self::Deleted => Ok(()), + Self::ExitStatus(status) => status.validate(), } } + + pub(crate) fn validate_for_protocol(&self, selected_version: u16) -> Result<()> { + self.validate()?; + if selected_version == 1 && matches!(self, Self::ExitStatus(_)) { + return Err(protocol_error( + ErrorCode::Unsupported, + "protocol version 1 cannot carry an exit-status response", + )); + } + Ok(()) + } } impl AgentHello { @@ -121,7 +163,8 @@ impl AgentHello { ), )); } - self.capabilities().validate() + self.capabilities() + .validate_for_protocol(self.selected_version()) } } @@ -142,7 +185,7 @@ impl RequestEnvelope { "agent request ID zero is reserved", )); } - self.request.validate() + self.request.validate_for_protocol(selected_version) } } @@ -167,7 +210,7 @@ impl ResponseEnvelope { )); } if let ResponseOutcome::Succeeded { response } = &self.outcome { - response.validate()?; + response.validate_for_protocol(selected_version)?; } Ok(()) } diff --git a/crates/agent/src/executor/init.rs b/crates/agent/src/executor/init.rs index 0c3f8d0..b3b401a 100644 --- a/crates/agent/src/executor/init.rs +++ b/crates/agent/src/executor/init.rs @@ -104,7 +104,8 @@ fn run_pid_namespace_init( match pid::fork_namespace_init() { Ok(ForkRole::Supervisor { child_pid }) => { drop(control); - pid::wait_for_child(child_pid) + let outcome = pid::wait_for_child(child_pid)?; + pid::mirror_child_outcome(outcome) } Ok(ForkRole::Init { runtime_pid }) => { if let Err(error) = prepare_create_environment(plan, bundle_directory, rootfs) { diff --git a/crates/agent/src/executor/mod.rs b/crates/agent/src/executor/mod.rs index 5d081da..f207766 100644 --- a/crates/agent/src/executor/mod.rs +++ b/crates/agent/src/executor/mod.rs @@ -13,16 +13,19 @@ mod rootfs; mod state; use std::path::{Path, PathBuf}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use a3s_oci_agent_protocol::{ AgentCapabilities, AgentCreateRequest, AgentDeleteRequest, AgentKillRequest, AgentStartRequest, - AgentState, AgentStateRequest, GuestAgentService, + AgentState, AgentStateRequest, AgentWaitRequest, GuestAgentService, }; use a3s_oci_sdk::oci_spec::runtime::ContainerState; -use a3s_oci_sdk::{async_trait, DeleteMode, Error, ErrorCode, OperationContext, Result}; +use a3s_oci_sdk::{ + async_trait, DeleteMode, Error, ErrorCode, ExitStatus, OperationContext, Result, +}; use tokio::io::AsyncWriteExt; use tokio::sync::Mutex; +use tokio::time::{sleep, Instant}; use crate::AGENT_VERSION; use pidfd::SignalOutcome; @@ -37,6 +40,7 @@ pub(crate) use pidfd::verify_support as verify_pidfd_support; const DEFAULT_RUNTIME_PARENT: &str = "/run"; const MAX_OPERATION_RECORDS: usize = 4_096; +const WAIT_POLL_INTERVAL: Duration = Duration::from_millis(10); /// Fail-closed Linux OCI executor shared by native and utility-VM drivers. #[derive(Debug)] @@ -147,7 +151,7 @@ impl LinuxExecutor { })?; Ok(Self { - capabilities: AgentCapabilities::core(AGENT_VERSION, std::env::consts::ARCH)?, + capabilities: AgentCapabilities::linux_executor(AGENT_VERSION, std::env::consts::ARCH)?, init_executable, runtime_root, state: Mutex::new(ExecutorState::default()), @@ -386,6 +390,51 @@ impl LinuxExecutor { state.containers.remove(&key); Ok(()) } + + async fn wait_new(&self, request: &AgentWaitRequest) -> Result { + let key = ContainerKey::from_target(&request.target)?; + let timeout = request.timeout_ms.map(Duration::from_millis); + let started = Instant::now(); + loop { + let status = { + let mut state = self.state.lock().await; + let record = state.containers.get_mut(&key).ok_or_else(|| { + executor_error( + ErrorCode::NotFound, + format!( + "container {} generation {} does not exist", + key.id, key.generation + ), + ) + })?; + record.poll_wait()? + }; + if let Some(status) = status { + return Ok(status); + } + + let delay = match timeout { + Some(limit) => { + let elapsed = started.elapsed(); + if elapsed >= limit { + return Err(executor_error( + ErrorCode::DeadlineExceeded, + format!( + "timed out after {} ms waiting for container {} generation {}", + request.timeout_ms.unwrap_or_default(), + key.id, + key.generation + ), + ) + .retryable(true)); + } + WAIT_POLL_INTERVAL.min(limit - elapsed) + } + None => WAIT_POLL_INTERVAL, + }; + sleep(delay).await; + } + } } #[async_trait] @@ -466,6 +515,10 @@ impl GuestAgentService for LinuxExecutor { ); result } + + async fn wait(&self, request: AgentWaitRequest) -> Result { + self.wait_new(&request).await + } } async fn create_private_directory(path: &Path) -> Result<()> { diff --git a/crates/agent/src/executor/pid.rs b/crates/agent/src/executor/pid.rs index c33dc35..d8f8631 100644 --- a/crates/agent/src/executor/pid.rs +++ b/crates/agent/src/executor/pid.rs @@ -11,6 +11,12 @@ pub(super) enum ForkRole { Init { runtime_pid: libc::pid_t }, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ChildOutcome { + Exited(i32), + Signaled(i32), +} + pub(super) fn fork_namespace_init() -> Result { let (mut supervisor_channel, mut init_channel) = UnixStream::pair().map_err(|error| { pid_error( @@ -64,14 +70,14 @@ pub(super) fn fork_namespace_init() -> Result { Ok(ForkRole::Supervisor { child_pid }) } -pub(super) fn wait_for_child(pid: libc::pid_t) -> Result<()> { +pub(super) fn wait_for_child(pid: libc::pid_t) -> Result { loop { let mut status = 0; // SAFETY: `status` points to writable storage and `pid` is the // positive child PID returned by `fork`. let waited = unsafe { libc::waitpid(pid, &mut status, 0) }; if waited == pid { - return Ok(()); + return decode_wait_status(status); } if waited < 0 && io::Error::last_os_error().kind() == io::ErrorKind::Interrupted { continue; @@ -80,6 +86,56 @@ pub(super) fn wait_for_child(pid: libc::pid_t) -> Result<()> { } } +pub(super) fn mirror_child_outcome(outcome: ChildOutcome) -> ! { + match outcome { + ChildOutcome::Exited(exit_code) => { + // SAFETY: `_exit` has no memory-safety preconditions and bypasses + // Rust destructors intentionally in this dedicated wrapper. + unsafe { libc::_exit(exit_code) } + } + ChildOutcome::Signaled(signal) => mirror_signal(signal), + } +} + +fn decode_wait_status(status: i32) -> Result { + if libc::WIFEXITED(status) { + return Ok(ChildOutcome::Exited(libc::WEXITSTATUS(status))); + } + if libc::WIFSIGNALED(status) { + return Ok(ChildOutcome::Signaled(libc::WTERMSIG(status))); + } + Err(pid_error( + ErrorCode::Internal, + format!("PID namespace init produced unsupported wait status {status:#x}"), + )) +} + +fn mirror_signal(signal: i32) -> ! { + // SIGKILL and SIGSTOP cannot be caught, ignored, or blocked. Other + // terminating signals are reset and unblocked so the supervisor has the + // same externally observable outcome as namespace PID 1. + if !matches!(signal, libc::SIGKILL | libc::SIGSTOP) { + // SAFETY: `signal` came from `waitpid`; SIG_DFL is a valid disposition. + unsafe { + libc::signal(signal, libc::SIG_DFL); + } + // SAFETY: the signal-set pointer is initialized before it is supplied + // to `sigprocmask`, and the old mask is intentionally not retained. + unsafe { + let mut set = std::mem::zeroed(); + libc::sigemptyset(&mut set); + libc::sigaddset(&mut set, signal); + libc::sigprocmask(libc::SIG_UNBLOCK, &set, std::ptr::null_mut()); + } + } + // SAFETY: `getpid` has no preconditions and `signal` is the positive value + // reported by `waitpid`. + unsafe { + libc::kill(libc::getpid(), signal); + libc::_exit(128_i32.saturating_add(signal).min(255)); + } +} + pub(super) async fn validate_runtime_pid( plan: &InitPlan, supervisor_pid: i32, @@ -247,7 +303,7 @@ fn pid_error(code: ErrorCode, message: impl Into) -> Error { #[cfg(test)] mod tests { - use super::{parse_pid_identity, PidIdentity}; + use super::{decode_wait_status, parse_pid_identity, ChildOutcome, PidIdentity}; #[test] fn parses_supervisor_and_nested_pid_namespace_identity() { @@ -272,4 +328,16 @@ mod tests { assert_eq!(error.code, a3s_oci_sdk::ErrorCode::FailedPrecondition); } } + + #[test] + fn decodes_normal_and_signal_wait_statuses() { + assert_eq!( + decode_wait_status(42 << 8).expect("normal exit status"), + ChildOutcome::Exited(42) + ); + assert_eq!( + decode_wait_status(libc::SIGKILL).expect("signal status"), + ChildOutcome::Signaled(libc::SIGKILL) + ); + } } diff --git a/crates/agent/src/executor/process.rs b/crates/agent/src/executor/process.rs index 1a8b19d..991f52c 100644 --- a/crates/agent/src/executor/process.rs +++ b/crates/agent/src/executor/process.rs @@ -1,12 +1,13 @@ use std::io; use std::os::linux::net::SocketAddrExt; use std::os::unix::net::{SocketAddr as StdSocketAddr, UnixListener as StdUnixListener}; +use std::os::unix::process::ExitStatusExt; use std::path::Path; -use std::process::{ExitStatus, Stdio}; +use std::process::{ExitStatus as ProcessExitStatus, Stdio}; use std::time::Duration; use a3s_oci_agent_protocol::AgentVsockEndpoint; -use a3s_oci_sdk::{Error, ErrorCode, Result}; +use a3s_oci_sdk::{Error, ErrorCode, ExitStatus, Result}; use tokio::io::AsyncWriteExt; use tokio::net::{UnixListener, UnixStream}; use tokio::process::{Child, Command}; @@ -25,6 +26,7 @@ pub(super) struct PreparedProcess { control: Option, pid: i32, pidfd: PidFd, + exit_status: Option, } impl PreparedProcess { @@ -71,7 +73,7 @@ impl PreparedProcess { enum ReadyOutcome { Connected(io::Result<(UnixStream, tokio::net::unix::SocketAddr)>), - Exited(io::Result), + Exited(io::Result), } let ready = timeout(INIT_READY_TIMEOUT, async { tokio::select! { @@ -167,6 +169,7 @@ impl PreparedProcess { control: Some(control), pid: runtime_pid, pidfd, + exit_status: None, }) } @@ -193,12 +196,18 @@ impl PreparedProcess { } pub(super) fn try_wait(&mut self) -> Result> { - self.child.try_wait().map_err(|error| { + if let Some(status) = &self.exit_status { + return Ok(Some(status.clone())); + } + let status = self.child.try_wait().map_err(|error| { process_error( ErrorCode::Internal, format!("failed to inspect container init state: {error}"), ) - }) + })?; + status + .map(|status| self.cache_exit_status(status)) + .transpose() } pub(super) fn signal(&self, signal: i32) -> Result { @@ -206,15 +215,8 @@ impl PreparedProcess { } pub(super) async fn force_stop(&mut self) -> Result<()> { - match self.child.try_wait() { - Ok(Some(_)) => return Ok(()), - Ok(None) => {} - Err(error) => { - return Err(process_error( - ErrorCode::Internal, - format!("failed to inspect container init before cleanup: {error}"), - )); - } + if self.try_wait()?.is_some() { + return Ok(()); } match self.pidfd.send_signal(libc::SIGKILL) { Ok(SignalOutcome::Delivered | SignalOutcome::Exited) => {} @@ -232,7 +234,9 @@ impl PreparedProcess { } } match timeout(INIT_READY_TIMEOUT, self.child.wait()).await { - Ok(Ok(_)) => {} + Ok(Ok(status)) => { + self.cache_exit_status(status)?; + } Ok(Err(error)) => { return Err(process_error( ErrorCode::Internal, @@ -249,6 +253,12 @@ impl PreparedProcess { } Ok(()) } + + fn cache_exit_status(&mut self, status: ProcessExitStatus) -> Result { + let status = convert_exit_status(status)?; + self.exit_status = Some(status.clone()); + Ok(status) + } } fn bind_control_listener() -> Result<(UnixListener, String)> { @@ -290,6 +300,19 @@ fn process_error(code: ErrorCode, message: impl Into) -> Error { Error::new(code, message).for_operation("run-container-init") } +fn convert_exit_status(status: ProcessExitStatus) -> Result { + if let Some(exit_code) = status.code() { + return ExitStatus::exited(exit_code); + } + if let Some(signal) = status.signal() { + return ExitStatus::signaled(signal, false); + } + Err(process_error( + ErrorCode::Internal, + format!("container init returned an unsupported process status {status}"), + )) +} + #[cfg(test)] mod tests { use std::io::Write; @@ -298,7 +321,10 @@ mod tests { use tokio::io::AsyncReadExt; - use super::bind_control_listener; + use std::os::unix::process::ExitStatusExt; + use std::process::ExitStatus as ProcessExitStatus; + + use super::{bind_control_listener, convert_exit_status}; use crate::executor::control::READY_BYTE; #[tokio::test(flavor = "current_thread")] @@ -325,4 +351,16 @@ mod tests { .expect("read ready byte"); assert_eq!(ready[0], READY_BYTE); } + + #[test] + fn converts_normal_and_signal_process_results() { + assert_eq!( + convert_exit_status(ProcessExitStatus::from_raw(42 << 8)).expect("normal result"), + a3s_oci_sdk::ExitStatus::exited(42).expect("normal SDK result") + ); + assert_eq!( + convert_exit_status(ProcessExitStatus::from_raw(libc::SIGKILL)).expect("signal result"), + a3s_oci_sdk::ExitStatus::signaled(libc::SIGKILL, false).expect("signal SDK result") + ); + } } diff --git a/crates/agent/src/executor/state.rs b/crates/agent/src/executor/state.rs index c64e989..1d8d52a 100644 --- a/crates/agent/src/executor/state.rs +++ b/crates/agent/src/executor/state.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use a3s_oci_agent_protocol::AgentState; use a3s_oci_sdk::oci_spec::runtime::ContainerState; -use a3s_oci_sdk::{ErrorCode, OperationId, Result}; +use a3s_oci_sdk::{ErrorCode, ExitStatus, OperationId, Result}; use serde::Serialize; use sha2::{Digest, Sha256}; @@ -110,10 +110,16 @@ pub(super) struct ContainerRecord { impl ContainerRecord { pub(super) fn refresh(&mut self) -> Result<()> { - if self.process.try_wait()?.is_some() { + self.poll_wait()?; + Ok(()) + } + + pub(super) fn poll_wait(&mut self) -> Result> { + let status = self.process.try_wait()?; + if status.is_some() { self.status = ContainerState::Stopped; } - Ok(()) + Ok(status) } pub(super) fn state(&self) -> Result { diff --git a/crates/cli/tests/cli.rs b/crates/cli/tests/cli.rs index b24cf67..7db4dc6 100644 --- a/crates/cli/tests/cli.rs +++ b/crates/cli/tests/cli.rs @@ -59,7 +59,7 @@ fn agent_vm_smoke_fails_closed_with_versioned_output() { assert_eq!(output.status.code(), Some(2)); let report: serde_json::Value = serde_json::from_slice(&output.stdout).expect("smoke output must be valid JSON"); - assert_eq!(report["schema_version"], "a3s.oci.agent-vm-smoke.v3"); + assert_eq!(report["schema_version"], "a3s.oci.agent-vm-smoke.v4"); assert_ne!(report["status"], "available"); if cfg!(all(target_os = "macos", target_arch = "aarch64")) { let cleanup = &report["macos_cleanup"]; @@ -110,7 +110,7 @@ fn native_linux_smoke_fails_closed_with_versioned_output() { assert_eq!(output.status.code(), Some(2)); let report: serde_json::Value = serde_json::from_slice(&output.stdout).expect("smoke output must be valid JSON"); - assert_eq!(report["schema_version"], "a3s.oci.native-linux-smoke.v1"); + assert_eq!(report["schema_version"], "a3s.oci.native-linux-smoke.v2"); assert_ne!(report["status"], "available"); } @@ -136,7 +136,7 @@ fn native_linux_multi_container_smoke_fails_closed_with_versioned_output() { serde_json::from_slice(&output.stdout).expect("diagnostic output must be valid JSON"); assert_eq!( report["schema_version"], - "a3s.oci.native-linux-multi-container-smoke.v1" + "a3s.oci.native-linux-multi-container-smoke.v2" ); assert_ne!(report["status"], "available"); } @@ -163,7 +163,7 @@ fn native_linux_fault_cleanup_fails_closed_with_versioned_output() { serde_json::from_slice(&output.stdout).expect("diagnostic output must be valid JSON"); assert_eq!( report["schema_version"], - "a3s.oci.native-linux-fault-cleanup.v1" + "a3s.oci.native-linux-fault-cleanup.v2" ); assert_eq!(report["lifecycle"]["requested_fault"], "after-start"); assert_ne!(report["status"], "available"); @@ -189,7 +189,7 @@ fn oci_vm_smoke_fails_closed_with_versioned_output() { assert_eq!(output.status.code(), Some(2)); let report: serde_json::Value = serde_json::from_slice(&output.stdout).expect("smoke output must be valid JSON"); - assert_eq!(report["schema_version"], "a3s.oci.oci-vm-smoke.v3"); + assert_eq!(report["schema_version"], "a3s.oci.oci-vm-smoke.v4"); assert_ne!(report["status"], "available"); } @@ -217,7 +217,7 @@ fn oci_vm_multi_container_smoke_fails_closed_with_versioned_output() { serde_json::from_slice(&output.stdout).expect("diagnostic output must be valid JSON"); assert_eq!( report["schema_version"], - "a3s.oci.oci-vm-multi-container-smoke.v1" + "a3s.oci.oci-vm-multi-container-smoke.v2" ); assert_ne!(report["status"], "available"); } @@ -244,7 +244,7 @@ fn oci_vm_fault_cleanup_fails_closed_with_versioned_output() { assert_eq!(output.status.code(), Some(2)); let report: serde_json::Value = serde_json::from_slice(&output.stdout).expect("diagnostic output must be valid JSON"); - assert_eq!(report["schema_version"], "a3s.oci.oci-vm-fault-cleanup.v1"); + assert_eq!(report["schema_version"], "a3s.oci.oci-vm-fault-cleanup.v2"); assert_eq!(report["lifecycle"]["requested_fault"], "after-kill"); assert_ne!(report["status"], "available"); } diff --git a/crates/runtime/src/agent_pipe.rs b/crates/runtime/src/agent_pipe.rs index 886b3da..0eae9d1 100644 --- a/crates/runtime/src/agent_pipe.rs +++ b/crates/runtime/src/agent_pipe.rs @@ -141,7 +141,7 @@ mod tests { use a3s_oci_agent_protocol::{ serve_agent_connection, AgentCapabilities, AgentClient, AgentCreateRequest, AgentDeleteRequest, AgentKillRequest, AgentStartRequest, AgentState, AgentStateRequest, - AgentVsockEndpoint, GuestAgentService, SessionToken, + AgentVsockEndpoint, GuestAgentService, SessionToken, AGENT_PROTOCOL_VERSION_MAX, }; use a3s_oci_sdk::{async_trait, Error, ErrorCode, Result}; use tokio::net::windows::named_pipe::ClientOptions; @@ -213,7 +213,10 @@ mod tests { let client = AgentClient::connect(stream, token) .await .expect("negotiate authenticated agent protocol"); - assert_eq!(client.hello().selected_version(), 1); + assert_eq!( + client.hello().selected_version(), + AGENT_PROTOCOL_VERSION_MAX + ); assert!(client.hello().capabilities().operations().is_empty()); drop(client); guest_task diff --git a/crates/runtime/src/agent_session.rs b/crates/runtime/src/agent_session.rs index 63bca14..7f370bf 100644 --- a/crates/runtime/src/agent_session.rs +++ b/crates/runtime/src/agent_session.rs @@ -237,7 +237,9 @@ impl AgentVmSession { return Some("guest selected an unexpected protocol version".into()); } if self.report.advertised_operations != expected_operations() { - return Some("guest agent did not advertise the exact core lifecycle".into()); + return Some( + "guest agent did not advertise the exact lifecycle and wait contract".into(), + ); } if self.report.agent_version.as_deref() != Some(env!("CARGO_PKG_VERSION")) { return Some("guest agent version does not match the host runtime version".into()); @@ -440,6 +442,7 @@ fn expected_operations() -> Vec { AgentOperation::Start, AgentOperation::Kill, AgentOperation::Delete, + AgentOperation::Wait, ] } diff --git a/crates/runtime/src/agent_socket/tests.rs b/crates/runtime/src/agent_socket/tests.rs index 756655a..e1ce04e 100644 --- a/crates/runtime/src/agent_socket/tests.rs +++ b/crates/runtime/src/agent_socket/tests.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use a3s_oci_agent_protocol::{ serve_agent_connection, AgentCapabilities, AgentClient, AgentCreateRequest, AgentDeleteRequest, AgentKillRequest, AgentOperation, AgentStartRequest, AgentState, AgentStateRequest, - AgentVsockEndpoint, GuestAgentService, SessionToken, + AgentVsockEndpoint, GuestAgentService, SessionToken, AGENT_PROTOCOL_VERSION_MAX, }; use a3s_oci_sdk::{async_trait, Error, ErrorCode, Result}; use tokio::process::{Child, Command}; @@ -108,7 +108,10 @@ async fn direct_child_negotiates_the_exact_authenticated_core_protocol() { let client = AgentClient::connect(stream, token) .await .expect("authenticate direct child agent"); - assert_eq!(client.hello().selected_version(), 1); + assert_eq!( + client.hello().selected_version(), + AGENT_PROTOCOL_VERSION_MAX + ); assert_eq!( client.hello().capabilities().operations(), [ diff --git a/crates/runtime/src/cleanup_report.rs b/crates/runtime/src/cleanup_report.rs index 83b00fb..499cf0c 100644 --- a/crates/runtime/src/cleanup_report.rs +++ b/crates/runtime/src/cleanup_report.rs @@ -7,9 +7,9 @@ use serde::{Deserialize, Serialize}; use crate::report::AgentVmSmokeReport; /// Schema emitted by the native Linux fault-cleanup diagnostic. -pub const NATIVE_LINUX_FAULT_CLEANUP_SCHEMA_VERSION: &str = "a3s.oci.native-linux-fault-cleanup.v1"; +pub const NATIVE_LINUX_FAULT_CLEANUP_SCHEMA_VERSION: &str = "a3s.oci.native-linux-fault-cleanup.v2"; /// Schema emitted by the utility-VM fault-cleanup diagnostic. -pub const OCI_VM_FAULT_CLEANUP_SCHEMA_VERSION: &str = "a3s.oci.oci-vm-fault-cleanup.v1"; +pub const OCI_VM_FAULT_CLEANUP_SCHEMA_VERSION: &str = "a3s.oci.oci-vm-fault-cleanup.v2"; /// Lifecycle boundary after which the cleanup diagnostic interrupts normal flow. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -172,6 +172,7 @@ impl NativeLinuxFaultCleanupReport { RuntimeOperation::Start, RuntimeOperation::Kill, RuntimeOperation::Delete, + RuntimeOperation::Wait, ] && self.lifecycle.is_success() && self.executor_shutdown_succeeded @@ -330,6 +331,7 @@ mod tests { RuntimeOperation::Start, RuntimeOperation::Kill, RuntimeOperation::Delete, + RuntimeOperation::Wait, ]; report.lifecycle = complete(LifecycleFaultPoint::AfterStart); report.executor_shutdown_succeeded = true; @@ -378,6 +380,7 @@ mod tests { AgentOperation::Start, AgentOperation::Kill, AgentOperation::Delete, + AgentOperation::Wait, ]; report.shim_report_verified = true; report.shim_exit_code = Some(0); diff --git a/crates/runtime/src/driver.rs b/crates/runtime/src/driver.rs index 1284ae2..86c68b0 100644 --- a/crates/runtime/src/driver.rs +++ b/crates/runtime/src/driver.rs @@ -1,10 +1,18 @@ use a3s_oci_core::DriverCapability; use a3s_oci_sdk::oci_spec::runtime::ContainerState; use a3s_oci_sdk::{ - async_trait, ContainerTarget, DeleteMode, Error, ErrorCode, IsolationRequest, OciBundle, - OperationContext, ProcessIo, Result, Signal, + async_trait, ContainerTarget, DeleteMode, Error, ErrorCode, ExitStatus, IsolationRequest, + OciBundle, OperationContext, ProcessIo, Result, RuntimeOperation, Signal, }; +const CORE_DRIVER_OPERATIONS: [RuntimeOperation; 5] = [ + RuntimeOperation::Create, + RuntimeOperation::State, + RuntimeOperation::Start, + RuntimeOperation::Kill, + RuntimeOperation::Delete, +]; + /// Driver-reported init-process state at one exact container generation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DriverState { @@ -110,6 +118,15 @@ pub struct DriverDeleteRequest { pub mode: DeleteMode, } +/// Exact init-process wait input passed to a driver. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DriverWaitRequest { + /// Container ID plus its exact generation. + pub target: ContainerTarget, + /// Maximum wait duration. `None` waits until the process terminates. + pub timeout_ms: Option, +} + /// Platform executor behind durable OCI lifecycle orchestration. /// /// Mutating calls must be idempotent by `OperationContext::operation_id`. @@ -126,6 +143,15 @@ pub trait RuntimeDriver: Send + Sync { /// Current availability, maturity, isolation, and probe evidence. fn capability(&self) -> DriverCapability; + /// Runtime operations implemented by this exact driver. + /// + /// The five core lifecycle operations are required by the current host + /// orchestrator. Optional operations must be advertised before the host + /// exposes or dispatches them. + fn operations(&self) -> &[RuntimeOperation] { + &CORE_DRIVER_OPERATIONS + } + /// Prepare all OCI create-time resources and return the blocked init PID. async fn create(&self, request: DriverCreateRequest) -> Result; @@ -140,4 +166,9 @@ pub trait RuntimeDriver: Send + Sync { /// Delete only resources owned by this container generation. async fn delete(&self, request: DriverDeleteRequest) -> Result<()>; + + /// Wait for the exact init process and return its stable terminal result. + async fn wait(&self, _request: DriverWaitRequest) -> Result { + Err(Error::unsupported("wait")) + } } diff --git a/crates/runtime/src/fault.rs b/crates/runtime/src/fault.rs index 2c5cff1..b02984c 100644 --- a/crates/runtime/src/fault.rs +++ b/crates/runtime/src/fault.rs @@ -146,17 +146,19 @@ pub(crate) enum DriverOperation { Start, Kill, Delete, + Wait, } impl DriverOperation { #[cfg(test)] - pub(crate) const ALL: [Self; 6] = [ + pub(crate) const ALL: [Self; 7] = [ Self::Capability, Self::Create, Self::State, Self::Start, Self::Kill, Self::Delete, + Self::Wait, ]; } diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 58a038c..e9a950d 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -43,7 +43,7 @@ pub use cleanup_report::{ }; pub use driver::{ DriverCreateRequest, DriverDeleteRequest, DriverKillRequest, DriverStartRequest, DriverState, - RuntimeDriver, + DriverWaitRequest, RuntimeDriver, }; pub use multi_container_report::{ MultiContainerLifecycleEvidence, NativeLinuxMultiContainerSmokeReport, diff --git a/crates/runtime/src/multi_container_report.rs b/crates/runtime/src/multi_container_report.rs index 90473ef..f77e6d8 100644 --- a/crates/runtime/src/multi_container_report.rs +++ b/crates/runtime/src/multi_container_report.rs @@ -1,14 +1,14 @@ use a3s_oci_core::{CapabilityStatus, HostPlatform}; -use a3s_oci_sdk::RuntimeOperation; +use a3s_oci_sdk::{ExitStatus, RuntimeOperation}; use serde::{Deserialize, Serialize}; use crate::AgentVmSmokeReport; /// Schema emitted by the native Linux multi-container lifecycle diagnostic. pub const NATIVE_LINUX_MULTI_CONTAINER_SCHEMA_VERSION: &str = - "a3s.oci.native-linux-multi-container-smoke.v1"; + "a3s.oci.native-linux-multi-container-smoke.v2"; /// Schema emitted by the utility-VM multi-container lifecycle diagnostic. -pub const OCI_VM_MULTI_CONTAINER_SCHEMA_VERSION: &str = "a3s.oci.oci-vm-multi-container-smoke.v1"; +pub const OCI_VM_MULTI_CONTAINER_SCHEMA_VERSION: &str = "a3s.oci.oci-vm-multi-container-smoke.v2"; /// Exact multi-container lifecycle and isolation evidence shared by both paths. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -48,10 +48,17 @@ pub struct MultiContainerLifecycleEvidence { pub b_unchanged_after_a_start: bool, /// Whether container B's marker remained absent after A started. pub marker_b_absent_after_a_start: bool, + /// Whether waiting on running container A left container B independently queryable. + pub wait_a_did_not_block_b: bool, /// Whether killing container A replayed exactly. pub kill_a_replayed: bool, /// Whether container A was observed stopped. pub a_stopped: bool, + /// Exact terminal result returned for container A. + #[serde(skip_serializing_if = "Option::is_none")] + pub wait_status_a: Option, + /// Whether repeated wait for container A returned the same result. + pub wait_a_replayed: bool, /// Whether container B remained at the same created barrier after A was killed. pub b_unchanged_after_a_kill: bool, /// Whether container B's marker remained absent after A was killed. @@ -84,6 +91,11 @@ pub struct MultiContainerLifecycleEvidence { pub kill_b_replayed: bool, /// Whether container B was observed stopped. pub b_stopped: bool, + /// Exact terminal result returned for container B. + #[serde(skip_serializing_if = "Option::is_none")] + pub wait_status_b: Option, + /// Whether repeated wait for container B returned the same result. + pub wait_b_replayed: bool, /// Whether deleting container B replayed exactly. pub delete_b_replayed: bool, /// Whether container B became unobservable after delete. @@ -109,8 +121,14 @@ impl MultiContainerLifecycleEvidence { && self.marker_a_verified && self.b_unchanged_after_a_start && self.marker_b_absent_after_a_start + && self.wait_a_did_not_block_b && self.kill_a_replayed && self.a_stopped + && self + .wait_status_a + .as_ref() + .is_some_and(|status| status.validate().is_ok()) + && self.wait_a_replayed && self.b_unchanged_after_a_kill && self.marker_b_absent_after_a_kill && self.delete_a_replayed @@ -127,6 +145,12 @@ impl MultiContainerLifecycleEvidence { && self.marker_b_verified && self.kill_b_replayed && self.b_stopped + && self + .wait_status_b + .as_ref() + .is_some_and(|status| status.validate().is_ok()) + && self.wait_status_a == self.wait_status_b + && self.wait_b_replayed && self.delete_b_replayed && self.b_missing_after_delete } @@ -203,8 +227,16 @@ impl NativeLinuxMultiContainerSmokeReport { RuntimeOperation::Start, RuntimeOperation::Kill, RuntimeOperation::Delete, + RuntimeOperation::Wait, ] && self.lifecycle.is_success() + && self.lifecycle.wait_status_a + == Some(ExitStatus { + exit_code: None, + signal: Some(9), + oom_killed: false, + }) + && self.lifecycle.wait_status_b == self.lifecycle.wait_status_a && self.markers_removed && self.executor_runtime_clean && self.session_root_clean @@ -278,6 +310,13 @@ impl OciVmMultiContainerSmokeReport { pub(crate) fn evidence_succeeded(&self) -> bool { self.bundles_loaded && self.lifecycle.is_success() + && self.lifecycle.wait_status_a + == Some(ExitStatus { + exit_code: Some(0), + signal: None, + oom_killed: false, + }) + && self.lifecycle.wait_status_b == self.lifecycle.wait_status_a && self.markers_removed && self.guest_runtime_clean && self.bridge.is_success() @@ -286,6 +325,8 @@ impl OciVmMultiContainerSmokeReport { #[cfg(test)] mod tests { + use a3s_oci_sdk::ExitStatus; + use super::MultiContainerLifecycleEvidence; #[test] @@ -315,8 +356,11 @@ mod tests { marker_a_verified: true, b_unchanged_after_a_start: true, marker_b_absent_after_a_start: true, + wait_a_did_not_block_b: true, kill_a_replayed: true, a_stopped: true, + wait_status_a: Some(ExitStatus::signaled(9, false).expect("exit status")), + wait_a_replayed: true, b_unchanged_after_a_kill: true, marker_b_absent_after_a_kill: true, delete_a_replayed: true, @@ -333,6 +377,8 @@ mod tests { marker_b_verified: true, kill_b_replayed: true, b_stopped: true, + wait_status_b: Some(ExitStatus::signaled(9, false).expect("exit status")), + wait_b_replayed: true, delete_b_replayed: true, b_missing_after_delete: true, } diff --git a/crates/runtime/src/native_linux_driver.rs b/crates/runtime/src/native_linux_driver.rs index 026ea39..8629222 100644 --- a/crates/runtime/src/native_linux_driver.rs +++ b/crates/runtime/src/native_linux_driver.rs @@ -4,17 +4,28 @@ use std::sync::Arc; use a3s_oci_agent::LinuxExecutor; use a3s_oci_agent_protocol::{ AgentBundle, AgentCreateRequest, AgentDeleteRequest, AgentKillRequest, AgentStartRequest, - AgentState, AgentStateRequest, GuestAgentService, GuestPath, + AgentState, AgentStateRequest, AgentWaitRequest, GuestAgentService, GuestPath, }; use a3s_oci_core::{CapabilityStatus, DriverCapability, DriverReadiness, IsolationClass}; use a3s_oci_sdk::oci_spec::runtime::ContainerState; -use a3s_oci_sdk::{async_trait, ContainerTarget, Error, ErrorCode, Result}; +use a3s_oci_sdk::{ + async_trait, ContainerTarget, Error, ErrorCode, ExitStatus, Result, RuntimeOperation, +}; use crate::driver::{ DriverCreateRequest, DriverDeleteRequest, DriverKillRequest, DriverStartRequest, DriverState, - RuntimeDriver, + DriverWaitRequest, RuntimeDriver, }; +const NATIVE_LINUX_OPERATIONS: [RuntimeOperation; 6] = [ + RuntimeOperation::Create, + RuntimeOperation::State, + RuntimeOperation::Start, + RuntimeOperation::Kill, + RuntimeOperation::Delete, + RuntimeOperation::Wait, +]; + /// Explicitly opted-in native Linux runtime driver. /// /// The default feature inventory remains `probe-only`. Constructing this @@ -84,6 +95,10 @@ impl RuntimeDriver for NativeLinuxDriver { self.capability.clone() } + fn operations(&self) -> &[RuntimeOperation] { + &NATIVE_LINUX_OPERATIONS + } + async fn create(&self, request: DriverCreateRequest) -> Result { if request.isolation.class() != IsolationClass::SharedHostKernel { return Err(Error::new( @@ -154,6 +169,15 @@ impl RuntimeDriver for NativeLinuxDriver { }) .await } + + async fn wait(&self, request: DriverWaitRequest) -> Result { + self.executor + .wait(AgentWaitRequest { + target: request.target, + timeout_ms: request.timeout_ms, + }) + .await + } } async fn guest_path(bundle: &Path) -> Result { diff --git a/crates/runtime/src/native_smoke/linux/lifecycle.rs b/crates/runtime/src/native_smoke/linux/lifecycle.rs index 4f90288..9d39ae8 100644 --- a/crates/runtime/src/native_smoke/linux/lifecycle.rs +++ b/crates/runtime/src/native_smoke/linux/lifecycle.rs @@ -5,8 +5,8 @@ use std::time::Duration; use a3s_oci_sdk::oci_spec::runtime::ContainerState; use a3s_oci_sdk::{ ContainerId, ContainerTarget, CreateRequest, DeleteMode, DeleteRequest, Error, ErrorCode, - IsolationRequest, KillRequest, OciBundle, OperationContext, OperationId, ProcessIo, - RuntimeClient, Signal, StartRequest, StateRequest, + ExitStatus, IsolationRequest, KillRequest, OciBundle, OperationContext, OperationId, ProcessIo, + RuntimeClient, Signal, StartRequest, StateRequest, WaitRequest, }; use tokio::time::{sleep, timeout, Instant}; @@ -88,6 +88,10 @@ pub(super) async fn exercise( return Err("native start did not leave the workload running".into()); } wait_for_marker(client, &target, marker, report).await?; + report.wait_timeout_enforced = wait_times_out_while_running(client, &target).await?; + if !report.wait_timeout_enforced { + return Err("native wait returned before the running init process exited".into()); + } let kill = KillRequest { context: operation(nonce, "kill")?, @@ -112,6 +116,27 @@ pub(super) async fn exercise( if !report.kill_replayed { return Err("native runtime did not exactly replay kill".into()); } + let wait = WaitRequest { + target: target.clone(), + timeout_ms: Some( + u64::try_from(LIFECYCLE_TIMEOUT.as_millis()) + .map_err(|_| "native lifecycle timeout does not fit wait request".to_string())?, + ), + }; + let waited = native_call("wait", client.wait(wait.clone())).await?; + let expected_exit = ExitStatus::signaled(libc::SIGKILL, false) + .map_err(|error| format!("failed to construct expected native exit status: {error}"))?; + report.wait_exit_status = Some(waited.clone()); + if waited != expected_exit { + return Err(format!( + "native wait returned {waited:?}, expected {expected_exit:?}" + )); + } + let replayed_wait = native_call("repeated wait", client.wait(wait)).await?; + report.wait_replayed = replayed_wait == waited; + if !report.wait_replayed { + return Err("native repeated wait returned a different exit status".into()); + } report.stopped_observed = wait_until_stopped(client, &target).await?; let delete = DeleteRequest { @@ -130,6 +155,28 @@ pub(super) async fn exercise( Ok(()) } +async fn wait_times_out_while_running( + client: &RuntimeClient, + target: &ContainerTarget, +) -> Result { + match timeout( + CALL_TIMEOUT, + client.wait(WaitRequest { + target: target.clone(), + timeout_ms: Some(50), + }), + ) + .await + { + Ok(Err(error)) if error.code == ErrorCode::DeadlineExceeded => Ok(true), + Ok(Err(error)) => Err(native_error("bounded wait while running", &error)), + Ok(Ok(status)) => Err(format!( + "bounded wait returned {status:?} while the native workload was running" + )), + Err(_) => Err("bounded native wait exceeded its outer call timeout".into()), + } +} + pub(super) async fn exercise_until_fault( client: &RuntimeClient, bundle: &OciBundle, diff --git a/crates/runtime/src/native_smoke/linux/multi_container/lifecycle.rs b/crates/runtime/src/native_smoke/linux/multi_container/lifecycle.rs index f355910..8db9440 100644 --- a/crates/runtime/src/native_smoke/linux/multi_container/lifecycle.rs +++ b/crates/runtime/src/native_smoke/linux/multi_container/lifecycle.rs @@ -5,8 +5,8 @@ use std::time::Duration; use a3s_oci_sdk::oci_spec::runtime::ContainerState; use a3s_oci_sdk::{ ContainerId, ContainerRecord, ContainerTarget, CreateRequest, DeleteMode, DeleteRequest, Error, - ErrorCode, IsolationRequest, KillRequest, OciBundle, OperationContext, OperationId, ProcessIo, - RuntimeClient, Signal, StartRequest, StateRequest, + ErrorCode, ExitStatus, IsolationRequest, KillRequest, OciBundle, OperationContext, OperationId, + ProcessIo, RuntimeClient, Signal, StartRequest, StateRequest, WaitRequest, }; use tokio::time::{sleep, timeout, Instant}; @@ -94,6 +94,12 @@ pub(super) async fn exercise( && report.lifecycle.marker_b_absent_after_a_start, "starting container A changed container B", )?; + report.lifecycle.wait_a_did_not_block_b = + wait_does_not_block_state(client, &target_a1, &target_b, &created_b).await?; + require( + report.lifecycle.wait_a_did_not_block_b, + "waiting on running container A blocked container B state", + )?; let kill_a = kill_request(nonce, "a-kill-1", target_a1.clone())?; let killed_a = native_call("kill container A", client.kill(kill_a.clone())).await?; @@ -104,6 +110,28 @@ pub(super) async fn exercise( report.lifecycle.kill_a_replayed, "container A kill replay changed its result", )?; + let waited_a = native_call( + "wait for container A", + client.wait(wait_request(target_a1.clone())), + ) + .await?; + let expected_exit = ExitStatus::signaled(libc::SIGKILL, false) + .map_err(|error| format!("failed to construct expected native exit status: {error}"))?; + require( + waited_a == expected_exit, + format!("container A wait returned {waited_a:?}, expected {expected_exit:?}"), + )?; + report.lifecycle.wait_status_a = Some(waited_a.clone()); + report.lifecycle.wait_a_replayed = native_call( + "repeat wait for container A", + client.wait(wait_request(target_a1.clone())), + ) + .await? + == waited_a; + require( + report.lifecycle.wait_a_replayed, + "container A repeated wait changed its result", + )?; report.lifecycle.a_stopped = wait_until_stopped(client, &target_a1).await?; report.lifecycle.b_unchanged_after_a_kill = state_equals(client, &target_b, &created_b, "container B after A kill").await?; @@ -229,6 +257,26 @@ pub(super) async fn exercise( report.lifecycle.kill_b_replayed, "container B kill replay changed its result", )?; + let waited_b = native_call( + "wait for container B", + client.wait(wait_request(target_b.clone())), + ) + .await?; + require( + waited_b == expected_exit, + format!("container B wait returned {waited_b:?}, expected {expected_exit:?}"), + )?; + report.lifecycle.wait_status_b = Some(waited_b.clone()); + report.lifecycle.wait_b_replayed = native_call( + "repeat wait for container B", + client.wait(wait_request(target_b.clone())), + ) + .await? + == waited_b; + require( + report.lifecycle.wait_b_replayed, + "container B repeated wait changed its result", + )?; report.lifecycle.b_stopped = wait_until_stopped(client, &target_b).await?; let delete_b = DeleteRequest { @@ -248,6 +296,49 @@ pub(super) async fn exercise( Ok(()) } +fn wait_request(target: ContainerTarget) -> WaitRequest { + WaitRequest { + target, + timeout_ms: Some(15_000), + } +} + +async fn wait_does_not_block_state( + client: &RuntimeClient, + waiting: &ContainerTarget, + observed: &ContainerTarget, + expected: &ContainerRecord, +) -> Result { + let wait = client.wait(WaitRequest { + target: waiting.clone(), + timeout_ms: Some(300), + }); + let state = async { + sleep(Duration::from_millis(50)).await; + timeout( + Duration::from_millis(200), + client.state(StateRequest { + target: observed.clone(), + }), + ) + .await + }; + let (wait_result, state_result) = tokio::join!(wait, state); + let wait_timed_out = + matches!(wait_result, Err(error) if error.code == ErrorCode::DeadlineExceeded); + let state_unchanged = match state_result { + Ok(Ok(record)) => &record == expected, + Ok(Err(error)) => { + return Err(native_error( + "container B state during container A wait", + &error, + )); + } + Err(_) => false, + }; + Ok(wait_timed_out && state_unchanged) +} + pub(super) async fn best_effort_delete(client: &RuntimeClient, nonce: &str) { for label in ["a", "b"] { let (Ok(id), Ok(context)) = ( diff --git a/crates/runtime/src/oci_smoke/utility_vm/lifecycle.rs b/crates/runtime/src/oci_smoke/utility_vm/lifecycle.rs index 2e3cd17..577c1df 100644 --- a/crates/runtime/src/oci_smoke/utility_vm/lifecycle.rs +++ b/crates/runtime/src/oci_smoke/utility_vm/lifecycle.rs @@ -4,11 +4,11 @@ use std::time::Duration; use a3s_oci_agent_protocol::{ AgentBundle, AgentClient, AgentCreateRequest, AgentDeleteRequest, AgentKillRequest, - AgentStartRequest, AgentStateRequest, GuestPath, + AgentStartRequest, AgentStateRequest, AgentWaitRequest, GuestPath, }; use a3s_oci_sdk::oci_spec::runtime::ContainerState; use a3s_oci_sdk::{ - ContainerTarget, DeleteMode, Error, ErrorCode, IoMode, OciBundle, OperationContext, + ContainerTarget, DeleteMode, Error, ErrorCode, ExitStatus, IoMode, OciBundle, OperationContext, OperationId, ProcessIo, Signal, }; use tokio::io::{AsyncRead, AsyncWrite}; @@ -85,6 +85,10 @@ pub(super) async fn exercise( } wait_for_running_marker(client, target, marker, report).await?; + report.wait_timeout_enforced = wait_times_out_while_running(client, target).await?; + if !report.wait_timeout_enforced { + return Err("guest wait returned before the running init process exited".into()); + } let kill = AgentKillRequest { context: operation(nonce, "kill")?, @@ -106,6 +110,26 @@ pub(super) async fn exercise( if !report.kill_replayed { return Err("guest did not exactly replay the kill result".into()); } + let wait = AgentWaitRequest { + target: target.clone(), + timeout_ms: Some( + u64::try_from(LIFECYCLE_TIMEOUT.as_millis()) + .map_err(|_| "guest lifecycle timeout does not fit wait request".to_string())?, + ), + }; + let waited = guest_call("wait", client.wait(wait.clone())).await?; + let expected_exit = ExitStatus::exited(0) + .map_err(|error| format!("failed to construct expected guest exit status: {error}"))?; + report.wait_exit_status = Some(waited.clone()); + if waited != expected_exit { + return Err(format!( + "guest wait returned {waited:?}, expected {expected_exit:?}" + )); + } + report.wait_replayed = guest_call("repeated wait", client.wait(wait)).await? == waited; + if !report.wait_replayed { + return Err("guest repeated wait returned a different exit status".into()); + } report.stopped_observed = wait_until_stopped(client, target).await?; let delete = AgentDeleteRequest { @@ -124,6 +148,28 @@ pub(super) async fn exercise( Ok(()) } +async fn wait_times_out_while_running( + client: &AgentClient, + target: &ContainerTarget, +) -> Result { + match timeout( + GUEST_CALL_TIMEOUT, + client.wait(AgentWaitRequest { + target: target.clone(), + timeout_ms: Some(50), + }), + ) + .await + { + Ok(Err(error)) if error.code == ErrorCode::DeadlineExceeded => Ok(true), + Ok(Err(error)) => Err(guest_error("bounded wait while running", &error)), + Ok(Ok(status)) => Err(format!( + "bounded wait returned {status:?} while the guest workload was running" + )), + Err(_) => Err("bounded guest wait exceeded its outer call timeout".into()), + } +} + pub(super) async fn exercise_until_fault( client: &AgentClient, bundle: &OciBundle, diff --git a/crates/runtime/src/oci_smoke/utility_vm/multi_container/lifecycle.rs b/crates/runtime/src/oci_smoke/utility_vm/multi_container/lifecycle.rs index cabd7f9..6356e40 100644 --- a/crates/runtime/src/oci_smoke/utility_vm/multi_container/lifecycle.rs +++ b/crates/runtime/src/oci_smoke/utility_vm/multi_container/lifecycle.rs @@ -4,12 +4,12 @@ use std::time::Duration; use a3s_oci_agent_protocol::{ AgentBundle, AgentClient, AgentCreateRequest, AgentDeleteRequest, AgentKillRequest, - AgentStartRequest, AgentState, AgentStateRequest, GuestPath, + AgentStartRequest, AgentState, AgentStateRequest, AgentWaitRequest, GuestPath, }; use a3s_oci_sdk::oci_spec::runtime::ContainerState; use a3s_oci_sdk::{ - ContainerId, ContainerTarget, DeleteMode, Error, ErrorCode, Generation, IoMode, OciBundle, - OperationContext, OperationId, ProcessIo, Signal, + ContainerId, ContainerTarget, DeleteMode, Error, ErrorCode, ExitStatus, Generation, IoMode, + OciBundle, OperationContext, OperationId, ProcessIo, Signal, }; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::time::{sleep, timeout, Instant}; @@ -109,6 +109,12 @@ pub(super) async fn exercise( && report.lifecycle.marker_b_absent_after_a_start, "starting container A changed container B", )?; + report.lifecycle.wait_a_did_not_block_b = + wait_does_not_block_state(client, &target_a1, &target_b, &created_b).await?; + require( + report.lifecycle.wait_a_did_not_block_b, + "waiting on running container A blocked container B state", + )?; let kill_a = kill_request(nonce, "a-kill-1", target_a1.clone())?; let killed_a = guest_call("kill container A", client.kill(kill_a.clone())).await?; @@ -119,6 +125,28 @@ pub(super) async fn exercise( report.lifecycle.kill_a_replayed, "container A kill replay changed its result", )?; + let waited_a = guest_call( + "wait for container A", + client.wait(wait_request(target_a1.clone())), + ) + .await?; + let expected_exit = ExitStatus::exited(0) + .map_err(|error| format!("failed to construct expected guest exit status: {error}"))?; + require( + waited_a == expected_exit, + format!("container A wait returned {waited_a:?}, expected {expected_exit:?}"), + )?; + report.lifecycle.wait_status_a = Some(waited_a.clone()); + report.lifecycle.wait_a_replayed = guest_call( + "repeat wait for container A", + client.wait(wait_request(target_a1.clone())), + ) + .await? + == waited_a; + require( + report.lifecycle.wait_a_replayed, + "container A repeated wait changed its result", + )?; report.lifecycle.a_stopped = wait_until_stopped(client, &target_a1).await?; report.lifecycle.b_unchanged_after_a_kill = state_equals(client, &target_b, &created_b, "container B after A kill").await?; @@ -247,6 +275,26 @@ pub(super) async fn exercise( report.lifecycle.kill_b_replayed, "container B kill replay changed its result", )?; + let waited_b = guest_call( + "wait for container B", + client.wait(wait_request(target_b.clone())), + ) + .await?; + require( + waited_b == expected_exit, + format!("container B wait returned {waited_b:?}, expected {expected_exit:?}"), + )?; + report.lifecycle.wait_status_b = Some(waited_b.clone()); + report.lifecycle.wait_b_replayed = guest_call( + "repeat wait for container B", + client.wait(wait_request(target_b.clone())), + ) + .await? + == waited_b; + require( + report.lifecycle.wait_b_replayed, + "container B repeated wait changed its result", + )?; report.lifecycle.b_stopped = wait_until_stopped(client, &target_b).await?; let delete_b = delete_request( @@ -267,6 +315,49 @@ pub(super) async fn exercise( Ok(()) } +fn wait_request(target: ContainerTarget) -> AgentWaitRequest { + AgentWaitRequest { + target, + timeout_ms: Some(15_000), + } +} + +async fn wait_does_not_block_state( + client: &AgentClient, + waiting: &ContainerTarget, + observed: &ContainerTarget, + expected: &AgentState, +) -> Result { + let wait = client.wait(AgentWaitRequest { + target: waiting.clone(), + timeout_ms: Some(300), + }); + let state = async { + sleep(Duration::from_millis(50)).await; + timeout( + Duration::from_millis(200), + client.state(AgentStateRequest { + target: observed.clone(), + }), + ) + .await + }; + let (wait_result, state_result) = tokio::join!(wait, state); + let wait_timed_out = + matches!(wait_result, Err(error) if error.code == ErrorCode::DeadlineExceeded); + let state_unchanged = match state_result { + Ok(Ok(state)) => &state == expected, + Ok(Err(error)) => { + return Err(guest_error( + "container B state during container A wait", + &error, + )); + } + Err(_) => false, + }; + Ok(wait_timed_out && state_unchanged) +} + pub(super) async fn best_effort_delete(client: &AgentClient, nonce: &str) { for label in ["a", "b"] { let (Ok(generation_two), Ok(context)) = ( diff --git a/crates/runtime/src/report.rs b/crates/runtime/src/report.rs index fd4a94d..0761365 100644 --- a/crates/runtime/src/report.rs +++ b/crates/runtime/src/report.rs @@ -1,7 +1,7 @@ use a3s_oci_agent_protocol::{AgentOperation, AGENT_PROTOCOL_VERSION_MAX}; use a3s_oci_core::CapabilityStatus; use a3s_oci_core::HostPlatform; -use a3s_oci_sdk::RuntimeOperation; +use a3s_oci_sdk::{ExitStatus, RuntimeOperation}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -10,11 +10,11 @@ pub const WHPX_SMOKE_SCHEMA_VERSION: &str = "a3s.oci.whpx-smoke.v1"; /// Schema emitted by the Hypervisor.framework VM-object smoke. pub const HVF_SMOKE_SCHEMA_VERSION: &str = "a3s.oci.hvf-smoke.v1"; /// Schema emitted by the authenticated guest-agent VM smoke. -pub const AGENT_VM_SMOKE_SCHEMA_VERSION: &str = "a3s.oci.agent-vm-smoke.v3"; +pub const AGENT_VM_SMOKE_SCHEMA_VERSION: &str = "a3s.oci.agent-vm-smoke.v4"; /// Schema emitted by the fixed OCI core-lifecycle utility-VM smoke. -pub const OCI_VM_SMOKE_SCHEMA_VERSION: &str = "a3s.oci.oci-vm-smoke.v3"; +pub const OCI_VM_SMOKE_SCHEMA_VERSION: &str = "a3s.oci.oci-vm-smoke.v4"; /// Schema emitted by the native Linux SDK lifecycle smoke. -pub const NATIVE_LINUX_SMOKE_SCHEMA_VERSION: &str = "a3s.oci.native-linux-smoke.v1"; +pub const NATIVE_LINUX_SMOKE_SCHEMA_VERSION: &str = "a3s.oci.native-linux-smoke.v2"; /// Result of querying WHPX and creating then deleting a partition object. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -320,6 +320,7 @@ impl AgentVmSmokeReport { AgentOperation::Start, AgentOperation::Kill, AgentOperation::Delete, + AgentOperation::Wait, ] && self.shim_report_verified && self.shim_exit_code == Some(0) @@ -363,6 +364,13 @@ pub struct NativeLinuxSmokeReport { pub kill_delivered: bool, /// Whether retrying kill replayed its exact original result. pub kill_replayed: bool, + /// Whether a bounded wait while running returned `deadline-exceeded`. + pub wait_timeout_enforced: bool, + /// Exact terminal result returned after SIGKILL. + #[serde(skip_serializing_if = "Option::is_none")] + pub wait_exit_status: Option, + /// Whether repeated wait returned the exact same terminal result. + pub wait_replayed: bool, /// Whether state eventually reported the workload stopped. pub stopped_observed: bool, /// Whether the workload produced the exact expected marker. @@ -402,6 +410,9 @@ impl NativeLinuxSmokeReport { running_observed: false, kill_delivered: false, kill_replayed: false, + wait_timeout_enforced: false, + wait_exit_status: None, + wait_replayed: false, stopped_observed: false, marker_verified: false, delete_succeeded: false, @@ -438,6 +449,7 @@ impl NativeLinuxSmokeReport { RuntimeOperation::Start, RuntimeOperation::Kill, RuntimeOperation::Delete, + RuntimeOperation::Wait, ] && self.dedicated_vm_rejected_before_create && self.create_returned_created @@ -448,6 +460,14 @@ impl NativeLinuxSmokeReport { && self.running_observed && self.kill_delivered && self.kill_replayed + && self.wait_timeout_enforced + && self.wait_exit_status + == Some(ExitStatus { + exit_code: None, + signal: Some(9), + oom_killed: false, + }) + && self.wait_replayed && self.stopped_observed && self.marker_verified && self.delete_succeeded @@ -487,6 +507,13 @@ pub struct OciVmSmokeReport { pub kill_delivered: bool, /// Whether retrying kill replayed its exact original result. pub kill_replayed: bool, + /// Whether a bounded wait while running returned `deadline-exceeded`. + pub wait_timeout_enforced: bool, + /// Exact terminal result returned after the configured SIGTERM trap. + #[serde(skip_serializing_if = "Option::is_none")] + pub wait_exit_status: Option, + /// Whether repeated wait returned the exact same terminal result. + pub wait_replayed: bool, /// Whether state eventually reported the workload stopped. pub stopped_observed: bool, /// Whether the workload produced the exact expected marker. @@ -523,6 +550,9 @@ impl OciVmSmokeReport { running_observed: false, kill_delivered: false, kill_replayed: false, + wait_timeout_enforced: false, + wait_exit_status: None, + wait_replayed: false, stopped_observed: false, marker_verified: false, delete_succeeded: false, @@ -565,6 +595,14 @@ impl OciVmSmokeReport { && self.running_observed && self.kill_delivered && self.kill_replayed + && self.wait_timeout_enforced + && self.wait_exit_status + == Some(ExitStatus { + exit_code: Some(0), + signal: None, + oom_killed: false, + }) + && self.wait_replayed && self.stopped_observed && self.marker_verified && self.delete_succeeded diff --git a/crates/runtime/src/service.rs b/crates/runtime/src/service.rs index 1aae627..cf5a69a 100644 --- a/crates/runtime/src/service.rs +++ b/crates/runtime/src/service.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::fmt; use std::path::Path; use std::sync::Arc; @@ -16,7 +16,8 @@ use a3s_oci_sdk::{ }; use crate::driver::{ - DriverCreateRequest, DriverDeleteRequest, DriverKillRequest, DriverStartRequest, RuntimeDriver, + DriverCreateRequest, DriverDeleteRequest, DriverKillRequest, DriverStartRequest, + DriverWaitRequest, RuntimeDriver, }; use crate::fault::{ DriverBoundaryStage, DriverOperation, FaultInjector, FaultPoint, NoFaultInjector, @@ -33,6 +34,7 @@ struct LifecycleHost { store: DurableStateStore, driver: Arc, capability: DriverCapability, + operations: BTreeSet, faults: Arc, } @@ -105,6 +107,7 @@ impl HostRuntimeService { ) .for_operation("open-host-runtime")); } + let operations = validate_driver_operations(driver.operations())?; let store = DurableStateStore::open_with_fault_injector(state_root, Arc::clone(&faults)).await?; Ok(Self { @@ -112,6 +115,7 @@ impl HostRuntimeService { store, driver, capability, + operations, faults, })), }) @@ -166,6 +170,14 @@ impl LifecycleHost { } } + fn ensure_operation(&self, operation: RuntimeOperation, name: &'static str) -> Result<()> { + if self.operations.contains(&operation) { + Ok(()) + } else { + Err(Error::unsupported(name)) + } + } + async fn fail_driver_operation( &self, operation_id: &a3s_oci_sdk::OperationId, @@ -179,6 +191,55 @@ impl LifecycleHost { } } +fn validate_driver_operations( + operations: &[RuntimeOperation], +) -> Result> { + const REQUIRED: [RuntimeOperation; 5] = [ + RuntimeOperation::Create, + RuntimeOperation::State, + RuntimeOperation::Start, + RuntimeOperation::Kill, + RuntimeOperation::Delete, + ]; + const HOST_SUPPORTED: [RuntimeOperation; 6] = [ + RuntimeOperation::Create, + RuntimeOperation::State, + RuntimeOperation::Start, + RuntimeOperation::Kill, + RuntimeOperation::Delete, + RuntimeOperation::Wait, + ]; + let reported = operations.iter().copied().collect::>(); + if reported.len() != operations.len() { + return Err(Error::new( + ErrorCode::FailedPrecondition, + "runtime driver advertises duplicate operations", + ) + .for_operation("open-host-runtime")); + } + if let Some(operation) = operations + .iter() + .find(|operation| !HOST_SUPPORTED.contains(operation)) + { + return Err(Error::new( + ErrorCode::FailedPrecondition, + format!("runtime driver advertises unsupported host operation {operation:?}"), + ) + .for_operation("open-host-runtime")); + } + if let Some(operation) = REQUIRED + .iter() + .find(|operation| !reported.contains(operation)) + { + return Err(Error::new( + ErrorCode::FailedPrecondition, + format!("runtime driver does not advertise required operation {operation:?}"), + ) + .for_operation("open-host-runtime")); + } + Ok(reported) +} + fn driver_state_error( operation: &'static str, expected: ContainerState, @@ -226,18 +287,10 @@ impl OciRuntimeService for HostRuntimeService { .for_operation("features") })?; - let operations = if self.lifecycle.is_some() { - vec![ - RuntimeOperation::Features, - RuntimeOperation::Create, - RuntimeOperation::State, - RuntimeOperation::Start, - RuntimeOperation::Kill, - RuntimeOperation::Delete, - ] - } else { - vec![RuntimeOperation::Features] - }; + let mut operations = vec![RuntimeOperation::Features]; + if let Some(lifecycle) = &self.lifecycle { + operations.extend(lifecycle.operations.iter().copied()); + } Ok(RuntimeInfo { oci, drivers: self.runtime_features(), @@ -434,8 +487,28 @@ impl OciRuntimeService for HostRuntimeService { Err(Error::unsupported("exec")) } - async fn wait(&self, _request: WaitRequest) -> Result { - Err(Error::unsupported("wait")) + async fn wait(&self, request: WaitRequest) -> Result { + let lifecycle = self.lifecycle("wait")?; + lifecycle.ensure_operation(RuntimeOperation::Wait, "wait")?; + request.validate()?; + let durable = lifecycle.store.state(&request.target).await?; + let target = ContainerTarget::exact(request.target.id, durable.generation); + lifecycle.driver_boundary(DriverOperation::Wait, DriverBoundaryStage::BeforeCall)?; + let result = lifecycle + .driver + .wait(DriverWaitRequest { + target: target.clone(), + timeout_ms: request.timeout_ms, + }) + .await; + lifecycle.driver_boundary(DriverOperation::Wait, DriverBoundaryStage::AfterCall)?; + let status = result?; + status.validate()?; + lifecycle + .store + .observe_state(&target, ContainerState::Stopped, None) + .await?; + Ok(status) } async fn list(&self, _request: ListRequest) -> Result> { diff --git a/crates/runtime/src/service/tests.rs b/crates/runtime/src/service/tests.rs index dcb1e00..9dca0d3 100644 --- a/crates/runtime/src/service/tests.rs +++ b/crates/runtime/src/service/tests.rs @@ -8,15 +8,15 @@ use a3s_oci_core::{ use a3s_oci_sdk::oci_spec::runtime::ContainerState; use a3s_oci_sdk::{ async_trait, ContainerId, ContainerTarget, CreateRequest, DeleteMode, DeleteRequest, Error, - ErrorCode, Generation, IsolationRequest, KillRequest, ListRequest, OciBundle, + ErrorCode, ExitStatus, Generation, IsolationRequest, KillRequest, ListRequest, OciBundle, OciRuntimeService, OperationContext, OperationId, ProcessIo, Result, RuntimeOperation, Signal, - StartRequest, StateRequest, TrustDomainId, + StartRequest, StateRequest, TrustDomainId, WaitRequest, }; use super::HostRuntimeService; use crate::{ DriverCreateRequest, DriverDeleteRequest, DriverKillRequest, DriverStartRequest, DriverState, - RuntimeDriver, + DriverWaitRequest, RuntimeDriver, }; mod fault_matrix; @@ -41,13 +41,16 @@ enum DriverCall { Start(DriverStartRequest), Kill(DriverKillRequest), Delete(DriverDeleteRequest), + Wait(DriverWaitRequest), } #[derive(Debug)] struct RecordingDriver { capability: DriverCapability, + operations: Vec, calls: Mutex>, states: Mutex>, + exits: Mutex>, failures: Mutex>>, } @@ -62,8 +65,17 @@ impl RecordingDriver { reason: None, evidence: BTreeMap::from([("test-driver".to_string(), "in-process".to_string())]), }, + operations: vec![ + RuntimeOperation::Create, + RuntimeOperation::State, + RuntimeOperation::Start, + RuntimeOperation::Kill, + RuntimeOperation::Delete, + RuntimeOperation::Wait, + ], calls: Mutex::new(Vec::new()), states: Mutex::new(HashMap::new()), + exits: Mutex::new(HashMap::new()), failures: Mutex::new(HashMap::new()), } } @@ -74,6 +86,12 @@ impl RecordingDriver { driver } + fn without_wait() -> Self { + let mut driver = Self::supported(); + driver.operations.pop(); + driver + } + fn calls(&self) -> Vec { self.calls.lock().expect("driver calls lock").clone() } @@ -113,6 +131,10 @@ impl RuntimeDriver for RecordingDriver { self.capability.clone() } + fn operations(&self) -> &[RuntimeOperation] { + &self.operations + } + async fn create(&self, request: DriverCreateRequest) -> Result { self.calls .lock() @@ -126,7 +148,11 @@ impl RuntimeDriver for RecordingDriver { self.states .lock() .expect("driver states lock") - .insert(request.target.id, (generation, state)); + .insert(request.target.id.clone(), (generation, state)); + self.exits + .lock() + .expect("driver exits lock") + .remove(&request.target.id); Ok(state) } @@ -180,7 +206,11 @@ impl RuntimeDriver for RecordingDriver { self.states .lock() .expect("driver states lock") - .insert(request.target.id, (generation, state)); + .insert(request.target.id.clone(), (generation, state)); + self.exits.lock().expect("driver exits lock").insert( + request.target.id, + ExitStatus::signaled(request.signal.get(), false)?, + ); Ok(state) } @@ -196,8 +226,53 @@ impl RuntimeDriver for RecordingDriver { .lock() .expect("driver states lock") .remove(&request.target.id); + self.exits + .lock() + .expect("driver exits lock") + .remove(&request.target.id); Ok(()) } + + async fn wait(&self, request: DriverWaitRequest) -> Result { + self.calls + .lock() + .expect("driver calls lock") + .push(DriverCall::Wait(request.clone())); + if let Some(error) = self.take_failure("wait") { + return Err(error); + } + let generation = Self::exact_generation(&request.target)?; + let states = self.states.lock().expect("driver states lock"); + let (actual_generation, state) = + states.get(&request.target.id).copied().ok_or_else(|| { + Error::new(ErrorCode::NotFound, "driver container does not exist") + .for_operation("driver-wait") + })?; + if generation != actual_generation { + return Err( + Error::new(ErrorCode::Conflict, "driver container generation mismatch") + .for_operation("driver-wait"), + ); + } + if state.status() != ContainerState::Stopped { + return Err(Error::new( + ErrorCode::DeadlineExceeded, + "driver process is still running", + ) + .for_operation("driver-wait") + .retryable(true)); + } + drop(states); + self.exits + .lock() + .expect("driver exits lock") + .get(&request.target.id) + .cloned() + .ok_or_else(|| { + Error::new(ErrorCode::Internal, "driver lost the init exit status") + .for_operation("driver-wait") + }) + } } fn identifier(value: &str, constructor: impl FnOnce(String) -> a3s_oci_sdk::Result) -> T { @@ -244,6 +319,55 @@ async fn reports_only_operations_that_are_currently_implemented() { assert_eq!(info.oci.oci_version_max(), "1.3.0"); } +#[tokio::test] +async fn rejects_invalid_driver_operation_inventories_before_opening_state() { + let temporary = tempfile::tempdir().expect("temporary directory"); + let inventories = [ + ( + "missing-core", + vec![ + RuntimeOperation::State, + RuntimeOperation::Start, + RuntimeOperation::Kill, + RuntimeOperation::Delete, + ], + ), + ( + "duplicate", + vec![ + RuntimeOperation::Create, + RuntimeOperation::State, + RuntimeOperation::Start, + RuntimeOperation::Kill, + RuntimeOperation::Delete, + RuntimeOperation::Delete, + ], + ), + ( + "unsupported", + vec![ + RuntimeOperation::Create, + RuntimeOperation::State, + RuntimeOperation::Start, + RuntimeOperation::Kill, + RuntimeOperation::Delete, + RuntimeOperation::Exec, + ], + ), + ]; + + for (name, operations) in inventories { + let root = temporary.path().join(name); + let mut driver = RecordingDriver::supported(); + driver.operations = operations; + let error = HostRuntimeService::open(&root, Arc::new(driver)) + .await + .expect_err("invalid driver operation inventory must fail"); + assert_eq!(error.code, ErrorCode::FailedPrecondition); + assert!(!root.exists(), "{name} created durable state"); + } +} + #[tokio::test] async fn incomplete_lifecycle_fails_explicitly() { let error = HostRuntimeService::new() @@ -273,6 +397,7 @@ async fn rust_sdk_lifecycle_is_durable_and_exactly_replayed() { RuntimeOperation::Start, RuntimeOperation::Kill, RuntimeOperation::Delete, + RuntimeOperation::Wait, ] ); @@ -315,6 +440,20 @@ async fn rust_sdk_lifecycle_is_durable_and_exactly_replayed() { assert_eq!(*stopped.state.status(), ContainerState::Stopped); assert_eq!(service.kill(kill).await.expect("replay kill"), stopped); + let wait = WaitRequest { + target: target.clone(), + timeout_ms: Some(1_000), + }; + let expected_exit = ExitStatus::signaled(15, false).expect("signal exit"); + assert_eq!( + service.wait(wait.clone()).await.expect("wait for init"), + expected_exit + ); + assert_eq!( + service.wait(wait).await.expect("repeat wait for init"), + expected_exit + ); + let delete = DeleteRequest { context: OperationContext::new(operation_id("delete-1")), target, @@ -352,6 +491,13 @@ async fn rust_sdk_lifecycle_is_durable_and_exactly_replayed() { .count(), 1 ); + assert_eq!( + calls + .iter() + .filter(|call| matches!(call, DriverCall::Wait(_))) + .count(), + 2 + ); let DriverCall::Create(driver_create) = &calls[0] else { panic!("create must be the first driver call"); }; @@ -367,6 +513,25 @@ async fn rust_sdk_lifecycle_is_durable_and_exactly_replayed() { assert_eq!(error.code, ErrorCode::NotFound); } +#[tokio::test] +async fn wait_is_exposed_only_when_the_driver_advertises_it() { + let temporary = tempfile::tempdir().expect("temporary directory"); + let driver = Arc::new(RecordingDriver::without_wait()); + let service = open_service(&temporary, driver).await; + let info = service.features().await.expect("configured features"); + assert!(!info.operations.contains(&RuntimeOperation::Wait)); + + let error = service + .wait(WaitRequest { + target: ContainerTarget::current(container_id("missing")), + timeout_ms: Some(0), + }) + .await + .expect_err("unadvertised wait must fail before state lookup"); + assert_eq!(error.code, ErrorCode::Unsupported); + assert_eq!(error.operation.as_deref(), Some("wait")); +} + #[tokio::test] async fn terminal_driver_failures_replay_and_release_container_claims() { let temporary = tempfile::tempdir().expect("temporary directory"); diff --git a/crates/runtime/src/service/tests/fault_matrix.rs b/crates/runtime/src/service/tests/fault_matrix.rs index 628a7f4..6065442 100644 --- a/crates/runtime/src/service/tests/fault_matrix.rs +++ b/crates/runtime/src/service/tests/fault_matrix.rs @@ -12,7 +12,7 @@ async fn every_host_driver_boundary_recovers_without_duplicate_effects() { let registry = FaultPoint::driver_registry(); assert_eq!( registry.len(), - 12, + 14, "update the host/driver fault contract when the registry changes" ); for point in registry { @@ -41,7 +41,10 @@ async fn exercise_driver_boundary(point: FaultPoint) { .await .expect("create setup container"); let target = ContainerTarget::exact(create.id.clone(), created.generation); - if matches!(operation, DriverOperation::Kill | DriverOperation::Delete) { + if matches!( + operation, + DriverOperation::Kill | DriverOperation::Delete | DriverOperation::Wait + ) { setup .start(StartRequest { context: OperationContext::new(operation_id("boundary-setup-start")), @@ -50,7 +53,7 @@ async fn exercise_driver_boundary(point: FaultPoint) { .await .expect("start setup container"); } - if operation == DriverOperation::Delete { + if matches!(operation, DriverOperation::Delete | DriverOperation::Wait) { setup .kill(KillRequest { context: OperationContext::new(operation_id("boundary-setup-kill")), @@ -130,6 +133,7 @@ const fn operation_requires_created_container(operation: DriverOperation) -> boo | DriverOperation::Start | DriverOperation::Kill | DriverOperation::Delete + | DriverOperation::Wait ) } @@ -141,6 +145,7 @@ const fn call_matches_operation(call: &DriverCall, operation: DriverOperation) - | (DriverCall::Start(_), DriverOperation::Start) | (DriverCall::Kill(_), DriverOperation::Kill) | (DriverCall::Delete(_), DriverOperation::Delete) + | (DriverCall::Wait(_), DriverOperation::Wait) ) } @@ -193,6 +198,15 @@ async fn invoke_operation( }) .await } + DriverOperation::Wait => { + service + .wait(WaitRequest { + target: target.expect("wait target").clone(), + timeout_ms: Some(1_000), + }) + .await?; + Ok(()) + } } } diff --git a/crates/sdk/src/model.rs b/crates/sdk/src/model.rs index c2065c7..ac58085 100644 --- a/crates/sdk/src/model.rs +++ b/crates/sdk/src/model.rs @@ -409,7 +409,7 @@ pub struct ProcessRecord { } /// Terminal process result. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ExitStatus { #[serde(skip_serializing_if = "Option::is_none")] pub exit_code: Option, @@ -418,6 +418,67 @@ pub struct ExitStatus { pub oom_killed: bool, } +impl<'de> Deserialize<'de> for ExitStatus { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Fields { + exit_code: Option, + signal: Option, + oom_killed: bool, + } + + let fields = Fields::deserialize(deserializer)?; + let status = Self { + exit_code: fields.exit_code, + signal: fields.signal, + oom_killed: fields.oom_killed, + }; + status.validate().map_err(de::Error::custom)?; + Ok(status) + } +} + +impl ExitStatus { + /// Construct a normal Linux process exit result. + pub fn exited(exit_code: i32) -> Result { + let status = Self { + exit_code: Some(exit_code), + signal: None, + oom_killed: false, + }; + status.validate()?; + Ok(status) + } + + /// Construct a Linux signal-termination result. + pub fn signaled(signal: i32, oom_killed: bool) -> Result { + let status = Self { + exit_code: None, + signal: Some(signal), + oom_killed, + }; + status.validate()?; + Ok(status) + } + + /// Validate the mutually exclusive terminal-result representation. + pub fn validate(&self) -> Result<()> { + match (self.exit_code, self.signal, self.oom_killed) { + (Some(exit_code), None, false) if (0..=255).contains(&exit_code) => Ok(()), + (None, Some(signal), _) if signal > 0 => Ok(()), + _ => Err(Error::new( + ErrorCode::InvalidArgument, + "exit status must contain either an exit code in 0..=255 or a positive signal; \ + oomKilled requires signal termination", + ) + .for_operation("validate-exit-status")), + } + } +} + /// Captured process output stream. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] @@ -507,7 +568,7 @@ pub struct EventBatch { #[cfg(test)] mod tests { - use super::Signal; + use super::{ExitStatus, Signal}; #[test] fn signal_deserialization_cannot_bypass_validation() { @@ -520,4 +581,52 @@ mod tests { assert!(serde_json::from_str::("0").is_err()); assert!(serde_json::from_str::("-9").is_err()); } + + #[test] + fn exit_status_requires_one_valid_terminal_outcome() { + assert_eq!( + ExitStatus::exited(42).expect("normal exit"), + ExitStatus { + exit_code: Some(42), + signal: None, + oom_killed: false, + } + ); + assert_eq!( + ExitStatus::signaled(9, true).expect("signal exit"), + ExitStatus { + exit_code: None, + signal: Some(9), + oom_killed: true, + } + ); + for status in [ + ExitStatus { + exit_code: None, + signal: None, + oom_killed: false, + }, + ExitStatus { + exit_code: Some(0), + signal: Some(9), + oom_killed: false, + }, + ExitStatus { + exit_code: Some(256), + signal: None, + oom_killed: false, + }, + ExitStatus { + exit_code: Some(1), + signal: None, + oom_killed: true, + }, + ] { + assert!(status.validate().is_err(), "{status:?} must be rejected"); + } + assert!(serde_json::from_str::( + r#"{"exit_code":0,"signal":9,"oom_killed":false}"# + ) + .is_err()); + } } diff --git a/docs/agent-protocol.md b/docs/agent-protocol.md index acdc7c8..04896f1 100644 --- a/docs/agent-protocol.md +++ b/docs/agent-protocol.md @@ -5,7 +5,7 @@ Linux guest executor. Windows WHPX, Linux KVM, and macOS HVF use the same messages. The crate does not expose libkrun, hypervisor, or guest details to A3S Box. -## Version 1 Contract +## Versioned Contract Before connection, the host calls `SessionToken::generate` to obtain a nonzero 256-bit token from the operating system's preferred random source and @@ -34,10 +34,20 @@ After negotiation: - cloned clients serialize requests on one connection. Protocol version 1 carries `create`, `state`, `start`, `kill`, and `delete`. -Every target includes a positive exact generation. Mutating guest operations -must be idempotent by `OperationId`. Production promotion also requires -recovery after an agent or host restart; the current bootstrap executor keeps -only session-local replay state. +Protocol version 2 preserves that contract and adds `wait`. Every target +includes a positive exact generation. A wait accepts an optional millisecond +timeout and returns exactly one terminal result: an exit code in `0..=255` or +a positive Linux signal, plus an OOM flag that is valid only for signal +termination. Repeated waits return the same cached result. + +The client breaks a long wait into bounded 25-millisecond guest requests. The +single correlated connection therefore remains available to query or control +another container between polls. A protocol-v1 peer neither advertises nor +accepts `wait`. + +Mutating guest operations must be idempotent by `OperationId`. Production +promotion also requires recovery after an agent or host restart; the current +bootstrap executor keeps only session-local replay state. ## Bundle Preservation @@ -63,7 +73,9 @@ bundle path. In-memory duplex tests cover: -- successful negotiation and the full core lifecycle; +- protocol-v1 negotiation and the unchanged five-operation core lifecycle; +- protocol-v2 wait with exact repeated signal status; +- rejection of a forged protocol-v1 wait before service dispatch; - two simultaneously registered container IDs with distinct PIDs, independent lifecycle transitions, stale-generation fencing, and generation-2 reuse; - wrong-token and incompatible-version rejection; @@ -89,15 +101,15 @@ step. The real WHPX `agent-vm-smoke` additionally boots the static musl Linux agent, carries its CID-host port 4093 connection through libkrun to that protected -pipe, authenticates the token, negotiates protocol version 1, and retains -bounded host and shim evidence. The current guest must advertise the exact -five core operations. +pipe, authenticates the token, negotiates protocol version 2, and retains +bounded host and shim evidence. The current guest must advertise the exact six +operations: `create`, `state`, `start`, `kill`, `delete`, and `wait`. The real macOS `agent-vm-smoke` builds the same agent as a static aarch64 musl binary, boots it through HVF, maps guest CID-host port 4093 to the verified Unix stream, and retains both the public shim PID and the direct VM worker PID -in `a3s.oci.agent-vm-smoke.v3`. The signed path must negotiate protocol version -1 and the exact five operations. The missing-entitlement path must exit with +in `a3s.oci.agent-vm-smoke.v4`. The signed path must negotiate protocol version +2 and the exact six operations. The missing-entitlement path must exit with status `2`, report no negotiation, terminate the shim process group, and leave no private endpoint residue. Both paths also retain in-process evidence that the exact runtime-owned endpoint was removed, the complete current-process @@ -107,9 +119,10 @@ VM-worker PID disappeared. The real Windows and macOS `oci-vm-smoke` paths keep the same authenticated connection open and prove a fixed bundle through create, state, exact create replay, start, running observation, marker verification, signal delivery, -exact kill replay, stopped observation, stopped-only delete, exact delete -replay, and a final NotFound state query. The marker proves that the workload -did not run before start and did run afterward. The init wrapper reads both +exact kill replay, a bounded wait while running, exact repeated terminal +status, stopped observation, stopped-only delete, exact delete replay, and a +final NotFound state query. The marker proves that the workload did not run +before start and did run afterward. The init wrapper reads both configured UTS names back before create returns, and the workload independently checks its hostname. When requested, the same create barrier also covers a new mount namespace, recursively private propagation, a self-bound rootfs, and @@ -127,9 +140,11 @@ runtime directory. The macOS `oci-vm-multi-container-smoke` path keeps two exact targets live on the same connection. It proves distinct runtime slots and PIDs, simultaneous create barriers, A/B transition isolation, session-local generation fencing, -exact operation replay, rejection of cross-container operation-ID reuse, and -independent pidfd-backed cleanup. Native Linux runs the equivalent sequence -through the durable SDK service around the same executor. +exact operation replay, rejection of cross-container operation-ID reuse, a +bounded wait on A that does not block B state, exact repeated terminal results +for both containers, and independent pidfd-backed cleanup. Native Linux runs +the equivalent sequence through the durable SDK service around the same +executor. The `oci-vm-fault-cleanup` companion stops after a successful create, start, or kill request and never sends delete. Session EOF must still make the agent call diff --git a/docs/guest-agent.md b/docs/guest-agent.md index 753899c..83aa14c 100644 --- a/docs/guest-agent.md +++ b/docs/guest-agent.md @@ -27,8 +27,9 @@ exact libkrun shim PID before it sends the token. ## Current Executor Boundary The current root-only bootstrap executor advertises -`create`, `state`, `start`, `kill`, and `delete`. It is intentionally narrower -than the final OCI executor and rejects every property it cannot enforce. +`create`, `state`, `start`, `kill`, `delete`, and `wait`. It is intentionally +narrower than the final OCI executor and rejects every property it cannot +enforce. The accepted bootstrap profile requires: @@ -95,9 +96,19 @@ through the retained pidfd, and delete supports stopped-only and force cleanup. Cleanup also signals through the pidfd and always reaps the authenticated wrapper before removing its runtime directory. Numeric PID reuse can therefore never redirect a lifecycle signal to an unrelated -process. Exact request retries are fingerprinted by `OperationId`, and reused -IDs with different requests fail. Generation fences remain in memory after -delete. +process. + +The PID-namespace supervisor preserves the configured namespace-PID-1 +process's terminal outcome: it exits with the same normal code or resets, +unblocks, and re-raises the same terminating signal. The executor converts +that raw Linux status into exactly one SDK exit code or signal, caches it per +generation, and returns it from every repeated init wait. A bounded wait +returns `DeadlineExceeded` while the process is still running, and the +executor releases its registry lock between observations so another +container remains independently queryable. + +Exact request retries are fingerprinted by `OperationId`, and reused IDs with +different requests fail. Generation fences remain in memory after delete. All guest registry, generation, and idempotency state is session-local. A closed host connection force-stops remaining init processes and removes the @@ -121,18 +132,20 @@ cargo zigbuild -p a3s-oci-agent --release ` ``` `a3s-oci agent-vm-smoke` proves the authenticated -guest-AF_VSOCK/libkrun/Windows-named-pipe path and verifies the exact core -operation advertisement. `a3s-oci oci-vm-smoke` additionally loads a bundle +guest-AF_VSOCK/libkrun/Windows-named-pipe path and verifies the exact +six-operation advertisement. `a3s-oci oci-vm-smoke` additionally loads a bundle below the VM rootfs and proves the distinct create/start barrier, state -observation, exact create/kill/delete replay, signal-driven stop, post-delete -NotFound, marker cleanup, and nominal guest runtime cleanup. +observation, exact create/kill/delete replay, bounded running wait, exact +repeated terminal status, signal-driven stop, post-delete NotFound, marker +cleanup, and nominal guest runtime cleanup. `a3s-oci oci-vm-multi-container-smoke` keeps two distinct bundle rootfs and runtime slots live behind the create barrier, proves that A's start, kill, -delete, recreation, stale generation, and replay conflicts do not alter B, -then completes B independently. The macOS HVF gate sends both init signals -through distinct retained pidfds and retains both per-container markers -together with guest-runtime and host-process cleanup evidence. +wait, delete, recreation, stale generation, and replay conflicts do not alter +or block B, then completes B independently. The macOS HVF gate sends both init +signals through distinct retained pidfds and retains both exact repeated exit +statuses and per-container markers together with guest-runtime and +host-process cleanup evidence. `a3s-oci oci-vm-fault-cleanup` stops after create, start, or kill, explicitly records that delete was not attempted, and requires guest executor shutdown to diff --git a/docs/linux-native.md b/docs/linux-native.md index 9801787..1821c06 100644 --- a/docs/linux-native.md +++ b/docs/linux-native.md @@ -80,11 +80,11 @@ submitted bundle is strictly loaded before the lifecycle begins, and the driver translates the durable host contract directly to the shared `LinuxExecutor`. -The versioned `a3s.oci.native-linux-smoke.v1` report requires all of the +The versioned `a3s.oci.native-linux-smoke.v2` report requires all of the following: 1. the service advertises exactly `features`, `create`, `state`, `start`, - `kill`, and `delete`; + `kill`, `delete`, and `wait`; 2. a dedicated-VM create fails as `Unsupported` before claiming the container ID or operation ID; 3. create returns a positive host-visible PID in the exact OCI `created` @@ -92,12 +92,16 @@ following: 4. the workload marker is absent before start; 5. retrying create replays its exact result; 6. start releases the prepared init and the marker is observed; -7. `SIGKILL` reaches the namespace PID 1 through its retained pidfd, and +7. a 50-millisecond wait returns `DeadlineExceeded` while the init process is + still running; +8. `SIGKILL` reaches the namespace PID 1 through its retained pidfd, and retrying kill replays its exact result; -8. state reaches `stopped`; -9. stopped-only delete and its exact retry succeed; -10. state returns `NotFound` after delete; -11. the marker, executor root, and complete smoke session are removed. +9. wait returns signal 9 with `oom_killed: false`, and a repeated wait returns + the same terminal result; +10. state reaches `stopped`; +11. stopped-only delete and its exact retry succeed; +12. state returns `NotFound` after delete; +13. the marker, executor root, and complete smoke session are removed. The smoke uses `SIGKILL` because Linux protects a PID-namespace init from default-action signals such as `SIGTERM`. General PID 1 supervision and signal @@ -126,14 +130,17 @@ executes both KVM-independent cases. shared `LinuxExecutor` for two distinct bundles. Both containers must return positive, different PIDs in `created` before either workload marker exists. Starting A must leave B's complete created record and marker unchanged; -killing and deleting A must do the same. +killing, waiting for, and deleting A must do the same. A bounded wait on the +running A must return `DeadlineExceeded` without preventing a concurrent state +query for B. After deleting A generation 1, the diagnostic removes only A's marker and recreates the same container ID. The durable host must allocate generation 2, reject an exact generation-1 state request, and reject reuse of A's create operation ID for B without changing B. Recreated A is force-deleted while B remains created, then B independently completes start, kill, stopped-only -delete, and post-delete `NotFound`. +delete, and post-delete `NotFound`. Both killed containers must return and +replay the exact signal-9 terminal result. Run it with a second bundle containing its own rootfs: @@ -145,11 +152,12 @@ sudo target/debug/a3s-oci native-linux-multi-container-smoke \ --work-parent "$work_parent" ``` -The `a3s.oci.native-linux-multi-container-smoke.v1` success additionally -requires exact create/start/kill/delete replay through each retained pidfd, -both marker removals, executor shutdown, and complete durable-session removal. -GitHub Actions runs the gate on x86_64 and aarch64 both without `/dev/kvm` and -with a present but unusable placeholder at that path. +The `a3s.oci.native-linux-multi-container-smoke.v2` success additionally +requires exact create/start/kill/delete replay, stable repeated wait results, +independent wait/state progress, both marker removals, executor shutdown, and +complete durable-session removal. GitHub Actions runs the gate on x86_64 and +aarch64 both without `/dev/kvm` and with a present but unusable placeholder at +that path. ## Fault-injected shutdown cleanup @@ -167,9 +175,10 @@ for fault in after-create after-start after-kill; do done ``` -The versioned `a3s.oci.native-linux-fault-cleanup.v1` report requires: +The versioned `a3s.oci.native-linux-fault-cleanup.v2` report requires: -1. the exact requested prefix and a positive runtime-visible init PID; +1. the exact seven-operation service inventory, requested prefix, and a + positive runtime-visible init PID; 2. marker absence behind create and exact marker contents after start; 3. `normal_delete_attempted: false`; 4. successful executor shutdown and disappearance of the init PID; @@ -189,7 +198,8 @@ The default driver must remain `probe-only` until at least the following pass: - namespace joins, time namespaces, and security-negative cases; - complete mount, credential, capability, seccomp, LSM, and cgroup v2 enforcement; -- init supervision, zombie reaping, exec, wait, and complete process I/O; +- namespace-internal init supervision and orphan/zombie reaping, exec, + per-process wait, and complete process I/O; - hooks, exhaustive durable-write and driver-error recovery injection, descriptor-relative path handling, and adversarial cleanup; - the complete A3S Box Rust, Python, and TypeScript Sandbox SDK suites on diff --git a/docs/macos-hvf.md b/docs/macos-hvf.md index f8c991a..8eb6c1b 100644 --- a/docs/macos-hvf.md +++ b/docs/macos-hvf.md @@ -281,8 +281,8 @@ The host runtime establishes the trust chain in this order: 8. remove the socket and private directory while retaining the accepted stream; 9. send the token only after process identity verification, negotiate protocol - version 1, and require the static arm64 guest to advertise exactly - `create`, `state`, `start`, `kill`, and `delete`. + version 2, and require the static arm64 guest to advertise exactly + `create`, `state`, `start`, `kill`, `delete`, and `wait`. The parent shim validates the rootfs, fixed `/usr/bin/a3s-oci-agent`, console, and protected socket before spawning the @@ -320,12 +320,12 @@ target/debug/a3s-oci agent-vm-smoke \ --console "$asset_dir/agent-console.log" ``` -The top-level report is `a3s.oci.agent-vm-smoke.v3`. A successful local Apple -Silicon qualification run retained the following contract: +The top-level report is `a3s.oci.agent-vm-smoke.v4`. A successful Apple Silicon +qualification must retain the following contract: ```json { - "schema_version": "a3s.oci.agent-vm-smoke.v3", + "schema_version": "a3s.oci.agent-vm-smoke.v4", "platform": "macos", "status": "available", "endpoint_bound": true, @@ -335,7 +335,7 @@ Silicon qualification run retained the following contract: "bridge_process_id": 12346, "shim_client_verified": true, "protocol_negotiated": true, - "selected_protocol": 1, + "selected_protocol": 2, "agent_version": "0.1.0", "guest_architecture": "aarch64", "advertised_operations": [ @@ -343,7 +343,8 @@ Silicon qualification run retained the following contract: "state", "start", "kill", - "delete" + "delete", + "wait" ], "shim_report_verified": true, "shim_exit_code": 0, @@ -404,15 +405,16 @@ target/debug/a3s-oci oci-vm-smoke \ --console "$asset_dir/oci-console.log" ``` -The signed Apple Silicon qualification completed all fields in -`a3s.oci.oci-vm-smoke.v3`: bundle loading, created state, exact create replay, -pre-start marker absence, start, running observation, exact kill replay, -stopped observation, marker verification, stopped-only delete, exact delete -replay, post-delete NotFound, marker removal, guest-runtime cleanup, and the -complete nested authenticated bridge report. The observed container PID was -positive, and the nested report proved exact endpoint removal, complete -current-process descriptor-inventory restoration, and disappearance of both -shim/worker PIDs after exit. +The signed Apple Silicon qualification contract is +`a3s.oci.oci-vm-smoke.v4`: bundle loading, created state, exact create replay, +pre-start marker absence, start, running observation, a bounded wait that must +time out while running, exact kill replay, exact normal exit status from the +SIGTERM trap, repeated wait, stopped observation, marker verification, +stopped-only delete, exact delete replay, post-delete NotFound, marker removal, +guest-runtime cleanup, and the complete nested authenticated bridge report. +The observed container PID must be positive, and the nested report must prove +exact endpoint removal, complete current-process descriptor-inventory +restoration, and disappearance of both shim/worker PIDs after exit. The guest opens a pidfd for the authenticated namespace PID 1 before returning created state. The lifecycle `SIGTERM` and all forced cleanup therefore target @@ -430,8 +432,9 @@ pre-entry behavior. `oci-vm-multi-container-smoke` submits two distinct contained bundles over one authenticated guest-agent connection. Both init processes must remain behind their create barriers with distinct positive guest-visible PIDs. Starting, -killing, and deleting A must preserve B's exact created state and leave B's -marker absent. +killing, waiting for, and deleting A must preserve B's exact created state and +leave B's marker absent. A bounded wait on running A must return +`DeadlineExceeded` without preventing a concurrent state query for B. The command then rejects A generation 1 after delete, recreates A as generation 2, rejects cross-container reuse of A's operation ID for B, removes recreated @@ -446,12 +449,13 @@ target/debug/a3s-oci oci-vm-multi-container-smoke \ --console "$asset_dir/oci-multi-container.log" ``` -The `a3s.oci.oci-vm-multi-container-smoke.v1` report also requires exact -mutation replay, both marker removals, no new guest runtime root, exact host -endpoint removal, shim and direct VM-worker reap, and full descriptor-inventory -restoration. The Apple Silicon HVF qualification and macOS CI both run this -gate; an unavailable-hypervisor branch must fail before negotiation while -retaining the same host cleanup evidence. +The `a3s.oci.oci-vm-multi-container-smoke.v2` report also requires exact +mutation replay, exact repeated normal-exit results for A and B, independent +wait/state progress, both marker removals, no new guest runtime root, exact +host endpoint removal, shim and direct VM-worker reap, and full +descriptor-inventory restoration. The Apple Silicon HVF qualification and +macOS CI both run this gate; an unavailable-hypervisor branch must fail before +negotiation while retaining the same host cleanup evidence. The pidfd requalification used the 8,493,136-byte static arm64 agent with SHA-256 @@ -478,13 +482,13 @@ for fault in after-create after-start after-kill; do done ``` -Each `a3s.oci.oci-vm-fault-cleanup.v1` success retains the exact requested and +Each `a3s.oci.oci-vm-fault-cleanup.v2` success retains the exact requested and injected boundary, a positive guest init PID, pre-start non-execution, and `normal_delete_attempted: false`. Closing the authenticated connection must then make the guest executor force-stop any live init and remove its runtime root before the agent and VM exit. -The nested `a3s.oci.agent-vm-smoke.v3` report independently requires exact +The nested `a3s.oci.agent-vm-smoke.v4` report independently requires exact endpoint removal, shim and direct VM-worker PID disappearance, and complete `(fd, fd_type)` inventory restoration. The outer report additionally requires marker removal and no new `a3s-oci-agent-*` directory under the guest `/run`. diff --git a/docs/oci-conformance.md b/docs/oci-conformance.md index dd178f7..4af7bcc 100644 --- a/docs/oci-conformance.md +++ b/docs/oci-conformance.md @@ -83,9 +83,9 @@ does not make it a native Windows container. | VM hypervisor, kernel, initrd, image, and parameters | Yes | Initial absolute-path and NUL rules; driver policy pending | No | No | | OCI `State` | Yes | Official schema, typed transitions, and generation fences | Durable core `creating`/`created`/`running`/`stopped` records | No | | OCI `Features` | Yes | Official schema, version and operation separation | Default service is feature-only; configured service reports only its implemented operations | No | -| `create/state/start/kill/delete` | SDK contract | Exhaustive request boundary, durable lifecycle tests, and fixed WHPX guest lifecycle smoke | Driver-independent orchestration plus a fail-closed guest bootstrap slice; no built-in launch-ready driver | No | +| `create/state/start/kill/delete` plus init `wait` | SDK contract | Exhaustive request boundary, durable lifecycle tests, protocol-v2 compatibility tests, and native/utility-VM lifecycle gates | Driver-independent core orchestration plus driver-advertised stable init wait and a fail-closed guest bootstrap slice; no built-in launch-ready driver | No | | Hooks and rollback ordering | SDK contract | Pending | No | No | -| Exec, I/O, PTY, wait, pause/resume, update | SDK contract | Typed requests | No | No | +| Exec, I/O, PTY, per-process wait, pause/resume, update | SDK contract | Typed requests | No | No | | Checkpoint and restore | SDK contract | Typed requests | No | No | The current runtime must therefore remain `probe-only`. diff --git a/docs/sdk-transport.md b/docs/sdk-transport.md index 45dc58c..7107278 100644 --- a/docs/sdk-transport.md +++ b/docs/sdk-transport.md @@ -68,6 +68,14 @@ durable bundle at both create and start. Its mutating methods are async, `Send + Sync`, and idempotent by `OperationId`. Platform resources and guest protocol types remain behind that boundary. +The host always requires the five core driver operations and advertises +`wait` only when the selected driver implements it. `WaitRequest` targets one +exact generation, accepts an optional millisecond timeout, and returns an +`ExitStatus` containing either an exit code in `0..=255` or a positive signal. +Repeated waits must return the same terminal result. The native Linux driver +and the protocol-v2 utility-VM guest path implement this contract; unsupported +drivers fail before dispatch. + ## Runtime Server Listener creation and access control belong to the runtime process because diff --git a/docs/windows-whpx.md b/docs/windows-whpx.md index ff2b43a..b8b99fc 100644 --- a/docs/windows-whpx.md +++ b/docs/windows-whpx.md @@ -29,10 +29,10 @@ The runtime: Linux rootfs, and verifies a guest-written marker through virtiofs; 9. boots `/usr/bin/a3s-oci-agent`, carries its host-CID port 4093 connection through libkrun to the protected pipe, authenticates the exact shim PID and - one-time token, negotiates protocol version 1, and waits for zero + one-time token, negotiates protocol version 2, and waits for zero guest/shim exit; -10. runs a fixed OCI bundle through distinct create, start, signal, and delete - calls, verifies lifecycle replay and cleanup, and keeps the built-in +10. runs a fixed OCI bundle through distinct create, start, signal, wait, and + delete calls, verifies lifecycle replay and cleanup, and keeps the built-in driver disabled; 11. emits stable JSON evidence through `a3s-oci features`, `a3s-oci whpx-smoke`, `a3s-oci-krun-shim context-smoke`, and @@ -93,9 +93,9 @@ A successful end-to-end agent VM smoke additionally proves that: - guest AF_VSOCK reaches the protected Windows named pipe through libkrun; - only the exact spawned shim PID is accepted before the token is sent; - the real guest authenticates the one-time token and negotiates protocol - version 1; + version 2; - the agent version and `x86_64` guest architecture are reported; -- the guest advertises exactly create, state, start, kill, and delete; +- the guest advertises exactly create, state, start, kill, delete, and wait; - the shim reports every VM configuration stage and a zero guest exit; - the host rejects an existing console destination rather than overwriting it. @@ -122,8 +122,9 @@ A successful fixed OCI VM smoke additionally proves that: - the wrapper applies the accepted rootfs, credentials, umask, and `no_new_privileges`, then calls `execve`; - the host observes `running` and the exact workload marker; -- kill delivers `SIGTERM`, its exact retry replays the original result, and - state then observes `stopped`; +- a bounded wait returns `DeadlineExceeded` while the workload is running; +- kill delivers `SIGTERM`, its exact retry replays the original result, wait + returns and replays exit code zero, and state then observes `stopped`; - stopped-only delete and its exact retry succeed; - state returns NotFound after delete; - the marker is removed and VM shutdown leaves no new agent runtime directory @@ -139,8 +140,10 @@ The fixed OCI lifecycle qualification used the 6,328,408-byte static musl agent with SHA-256 `4b21a230d4183abe053823a63893f5ab0663c118811c81229bdfba0816fc9b81`. Its report selected protocol version 1, identified the guest as `x86_64`, -verified every fixed lifecycle field, retained the complete successful shim -report, and returned exit status zero. +verified every then-current fixed lifecycle field, retained the complete +successful shim report, and returned exit status zero. This historical run +predates the protocol-v2 wait gate and does not qualify the current report +schemas. The UTS qualification configured hostname `a3s-smoke` and domainname `runtime.test`, checked the hostname from the workload, and crossed the create @@ -199,7 +202,7 @@ The next vertical slice must: 2. mount one protected runtime-owned root through virtio-fs; 3. add user/time namespaces, namespace joins, advanced mount, capability, resource, seccomp, and hook enforcement; -4. return stdout, stderr, and the natural exit code; +4. return stdout, stderr, and arbitrary natural exit codes; 5. reconcile stopped state after host runtime restart; 6. add concurrent-container and negative isolation evidence; 7. prove cleanup under fault injection and repeated soak runs.