feat(runtime): prove multi-container lifecycle isolation

Author: Roy Lin

.github/scripts/native-linux-smoke.sh

@@ -11,11 +11,14 @@ sudo apt-get install --yes busybox-static jq
 cargo build -p a3s-oci-agent -p a3s-oci-cli
 
 bundle="$runner_temp/a3s-native-bundle"
+bundle_b="$runner_temp/a3s-native-bundle-b"
 work_parent="$runner_temp/a3s-native-work"
-mkdir -p "$bundle/rootfs/bin" "$work_parent"
-cp fixtures/native-linux/config.json "$bundle/config.json"
-cp "$(command -v busybox)" "$bundle/rootfs/bin/busybox"
-ln -s busybox "$bundle/rootfs/bin/sh"
+mkdir -p "$bundle/rootfs/bin" "$bundle_b/rootfs/bin" "$work_parent"
+for candidate in "$bundle" "$bundle_b"; do
+  cp fixtures/native-linux/config.json "$candidate/config.json"
+  cp "$(command -v busybox)" "$candidate/rootfs/bin/busybox"
+  ln -s busybox "$candidate/rootfs/bin/sh"
+done
 
 run_smoke() {
   local expected_kvm_present="$1"
@@ -31,6 +34,69 @@ run_smoke() {
     <<<"$output" >/dev/null
 }
 
+run_multi_container_smoke() {
+  local expected_kvm_present="$1"
+  local output
+  output="$(sudo "$PWD/target/debug/a3s-oci" \
+    native-linux-multi-container-smoke \
+    --agent "$PWD/target/debug/a3s-oci-agent" \
+    --bundle-a "$bundle" \
+    --bundle-b "$bundle_b" \
+    --work-parent "$work_parent")"
+  printf '%s\n' "$output"
+  jq --exit-status \
+    --argjson expected "$expected_kvm_present" \
+    '.schema_version == "a3s.oci.native-linux-multi-container-smoke.v1"
+     and .platform == "linux" and .status == "available"
+     and .kvm_device_present == $expected
+     and .bundles_loaded
+     and .service_operations
+         == ["features", "create", "state", "start", "kill", "delete"]
+     and .lifecycle.distinct_bundle_directories
+     and .lifecycle.distinct_rootfs_directories
+     and .lifecycle.both_created_before_start
+     and .lifecycle.initial_generation_a == 1
+     and .lifecycle.initial_generation_b == 1
+     and .lifecycle.recreated_generation_a == 2
+     and (.lifecycle.created_pid_a > 0)
+     and (.lifecycle.created_pid_b > 0)
+     and .lifecycle.distinct_created_pids
+     and .lifecycle.create_replays_exact
+     and .lifecycle.both_markers_absent_before_start
+     and .lifecycle.start_a_replayed
+     and .lifecycle.marker_a_verified
+     and .lifecycle.b_unchanged_after_a_start
+     and .lifecycle.marker_b_absent_after_a_start
+     and .lifecycle.kill_a_replayed
+     and .lifecycle.a_stopped
+     and .lifecycle.b_unchanged_after_a_kill
+     and .lifecycle.marker_b_absent_after_a_kill
+     and .lifecycle.delete_a_replayed
+     and .lifecycle.a_missing_after_delete
+     and .lifecycle.b_unchanged_after_a_delete
+     and .lifecycle.stale_generation_rejected
+     and .lifecycle.generation_a_monotonic
+     and .lifecycle.recreate_a_replayed
+     and .lifecycle.marker_a_absent_after_recreate
+     and .lifecycle.cross_container_operation_rejected
+     and .lifecycle.b_unchanged_after_replay_conflict
+     and .lifecycle.recreated_a_deleted
+     and .lifecycle.start_b_replayed
+     and .lifecycle.marker_b_verified
+     and .lifecycle.kill_b_replayed
+     and .lifecycle.b_stopped
+     and .lifecycle.delete_b_replayed
+     and .lifecycle.b_missing_after_delete
+     and .markers_removed
+     and .executor_runtime_clean
+     and .session_root_clean
+     and (.reason == null)' \
+    <<<"$output" >/dev/null
+  test ! -e "$bundle/rootfs/.a3s-oci-native-smoke"
+  test ! -e "$bundle_b/rootfs/.a3s-oci-native-smoke"
+  test -z "$(find "$work_parent" -mindepth 1 -print -quit)"
+}
+
 run_fault_cleanup() {
   local phase
   local output
@@ -94,6 +160,8 @@ if [[ -e /dev/kvm || -L /dev/kvm ]]; then
 fi
 
 run_smoke false
+run_multi_container_smoke false
 run_fault_cleanup
 sudo mkdir /dev/kvm
 run_smoke true
+run_multi_container_smoke true

.github/workflows/ci.yml

@@ -449,6 +449,136 @@ jobs:
           )"
           test "$endpoint_after" = "$endpoint_baseline"
           test ! -e "$marker"
+      - if: runner.os == 'macOS'
+        name: Multi-container macOS OCI lifecycle or fail-closed worker
+        shell: bash
+        run: |
+          rootfs_dir="$RUNNER_TEMP/a3s-oci-alpine-aarch64"
+          rootfs_archive="$RUNNER_TEMP/alpine-minirootfs-3.22.5-aarch64.tar.gz"
+          signed_dir="$RUNNER_TEMP/a3s-oci-agent-vm-signed"
+          bundle_a="$rootfs_dir/var/lib/a3s-oci-smoke/bundle"
+          bundle_b="$rootfs_dir/var/lib/a3s-oci-smoke/bundle-b"
+          marker_a="$bundle_a/rootfs/.a3s-oci-create-start-smoke"
+          marker_b="$bundle_b/rootfs/.a3s-oci-create-start-smoke"
+          console="$RUNNER_TEMP/a3s-oci-multi-container.log"
+          mkdir -p "$bundle_b/rootfs"
+          cp fixtures/utility-vm/config.json "$bundle_b/config.json"
+          tar -xzf "$rootfs_archive" -C "$bundle_b/rootfs"
+
+          endpoint_baseline="$(
+            find /private/tmp -maxdepth 1 \
+              -name 'a3s-oci-agent-*' -print | sort
+          )"
+          runtime_baseline="$(
+            find "$rootfs_dir/run" -maxdepth 1 \
+              -name 'a3s-oci-agent-*' -print | sort
+          )"
+          set +e
+          output="$(
+            target/debug/a3s-oci oci-vm-multi-container-smoke \
+              --shim "$signed_dir/a3s-oci-krun-shim" \
+              --vm-rootfs "$rootfs_dir" \
+              --bundle-a "$bundle_a" \
+              --bundle-b "$bundle_b" \
+              --console "$console"
+          )"
+          status=$?
+          set -e
+          printf '%s\n' "$output"
+
+          support="$(sysctl -n kern.hv_support 2>/dev/null || printf unavailable)"
+          if [[ "$support" == "1" ]]; then
+            test "$status" -eq 0
+            jq --exit-status \
+              '.schema_version == "a3s.oci.oci-vm-multi-container-smoke.v1"
+               and .platform == "macos" and .status == "available"
+               and .bundles_loaded
+               and .lifecycle.distinct_bundle_directories
+               and .lifecycle.distinct_rootfs_directories
+               and .lifecycle.both_created_before_start
+               and .lifecycle.initial_generation_a == 1
+               and .lifecycle.initial_generation_b == 1
+               and .lifecycle.recreated_generation_a == 2
+               and (.lifecycle.created_pid_a > 0)
+               and (.lifecycle.created_pid_b > 0)
+               and .lifecycle.distinct_created_pids
+               and .lifecycle.create_replays_exact
+               and .lifecycle.both_markers_absent_before_start
+               and .lifecycle.start_a_replayed
+               and .lifecycle.marker_a_verified
+               and .lifecycle.b_unchanged_after_a_start
+               and .lifecycle.marker_b_absent_after_a_start
+               and .lifecycle.kill_a_replayed
+               and .lifecycle.a_stopped
+               and .lifecycle.b_unchanged_after_a_kill
+               and .lifecycle.marker_b_absent_after_a_kill
+               and .lifecycle.delete_a_replayed
+               and .lifecycle.a_missing_after_delete
+               and .lifecycle.b_unchanged_after_a_delete
+               and .lifecycle.stale_generation_rejected
+               and .lifecycle.generation_a_monotonic
+               and .lifecycle.recreate_a_replayed
+               and .lifecycle.marker_a_absent_after_recreate
+               and .lifecycle.cross_container_operation_rejected
+               and .lifecycle.b_unchanged_after_replay_conflict
+               and .lifecycle.recreated_a_deleted
+               and .lifecycle.start_b_replayed
+               and .lifecycle.marker_b_verified
+               and .lifecycle.kill_b_replayed
+               and .lifecycle.b_stopped
+               and .lifecycle.delete_b_replayed
+               and .lifecycle.b_missing_after_delete
+               and .markers_removed and .guest_runtime_clean
+               and .bridge.platform == "macos"
+               and .bridge.status == "available"
+               and .bridge.protocol_negotiated
+               and .bridge.advertised_operations
+                   == ["create", "state", "start", "kill", "delete"]
+               and .bridge.shim_report_verified
+               and .bridge.shim_exit_code == 0
+               and .bridge.macos_cleanup.endpoint_removed
+               and .bridge.macos_cleanup.shim_reaped
+               and .bridge.macos_cleanup.bridge_reaped
+               and (.bridge.macos_cleanup.open_descriptors_before > 0)
+               and (.bridge.macos_cleanup.open_descriptors_before
+                    == .bridge.macos_cleanup.open_descriptors_after)
+               and .bridge.macos_cleanup.descriptor_inventory_restored
+               and (.bridge.macos_cleanup.reason == null)
+               and (.reason == null)' \
+              <<<"$output" >/dev/null
+          else
+            test "$status" -eq 2
+            jq --exit-status \
+              '.schema_version == "a3s.oci.oci-vm-multi-container-smoke.v1"
+               and .platform == "macos" and .status == "unavailable"
+               and .bundles_loaded
+               and .bridge.platform == "macos"
+               and .bridge.status == "unavailable"
+               and (.bridge.protocol_negotiated | not)
+               and .bridge.macos_cleanup.endpoint_removed
+               and .bridge.macos_cleanup.shim_reaped
+               and .bridge.macos_cleanup.bridge_reaped
+               and (.bridge.macos_cleanup.open_descriptors_before > 0)
+               and (.bridge.macos_cleanup.open_descriptors_before
+                    == .bridge.macos_cleanup.open_descriptors_after)
+               and .bridge.macos_cleanup.descriptor_inventory_restored
+               and (.bridge.macos_cleanup.reason == null)
+               and (.reason | length > 0)' \
+              <<<"$output" >/dev/null
+          fi
+
+          endpoint_after="$(
+            find /private/tmp -maxdepth 1 \
+              -name 'a3s-oci-agent-*' -print | sort
+          )"
+          runtime_after="$(
+            find "$rootfs_dir/run" -maxdepth 1 \
+              -name 'a3s-oci-agent-*' -print | sort
+          )"
+          test "$endpoint_after" = "$endpoint_baseline"
+          test "$runtime_after" = "$runtime_baseline"
+          test ! -e "$marker_a"
+          test ! -e "$marker_b"
       - if: runner.os == 'macOS'
         name: Fault-injected macOS OCI cleanup
         shell: bash

README.md

@@ -84,7 +84,8 @@ Workload calls require an explicitly supplied launch-ready `RuntimeDriver`.
   quarantine, backed by exhaustive durable-write and driver-boundary fault
   matrices
 - **Shared Linux Executor**: Reuse one fail-closed namespace, mount, process,
-  and cleanup implementation directly on Linux and through the guest agent
+  and cleanup implementation directly on Linux and through the guest agent,
+  with independently fenced per-container generations
 - **Cross-Platform Drivers**: Inspect native Linux, KVM, HVF, and WHPX
   prerequisites without silently weakening requested isolation
 - **Typed SDK and IPC**: Expose an async `Send + Sync` Rust contract with
@@ -163,6 +164,27 @@ observation, idempotent mutation replay, marker verification, post-delete
 [Native Linux Development](docs/linux-native.md) for the accepted profile and
 remaining production gates.
 
+Create a second distinct bundle to run the multi-container gate:
+
+```sh
+bundle_b="$demo_root/bundle-b"
+mkdir -p "$bundle_b/rootfs/bin"
+cp fixtures/native-linux/config.json "$bundle_b/config.json"
+cp "$(command -v busybox)" "$bundle_b/rootfs/bin/busybox"
+ln -s busybox "$bundle_b/rootfs/bin/sh"
+
+sudo target/debug/a3s-oci native-linux-multi-container-smoke \
+  --agent "$PWD/target/debug/a3s-oci-agent" \
+  --bundle-a "$bundle" \
+  --bundle-b "$bundle_b" \
+  --work-parent "$work_parent"
+```
+
+The versioned report retains two simultaneous create barriers, distinct
+positive PIDs, operation replay isolation, A/B lifecycle independence,
+generation-1 rejection after A is recreated as generation 2, and complete
+process, marker, executor-root, and durable-session cleanup.
+
 The separate fault-cleanup diagnostic deliberately stops before OCI delete and
 requires executor shutdown to reclaim the live process and all scoped state:
 
@@ -300,6 +322,29 @@ running and stopped observation, marker verification, post-delete `NotFound`,
 and nominal endpoint, process, marker, and guest-runtime cleanup. This remains
 a fixed development profile rather than an arbitrary-workload driver.
 
+The multi-container gate places two distinct bundles in the same guest and
+keeps both init processes behind the create barrier:
+
+```sh
+bundle_b="$rootfs_dir/rootfs/var/lib/a3s-oci-smoke/bundle-b"
+mkdir -p "$bundle_b/rootfs"
+cp fixtures/utility-vm/config.json "$bundle_b/config.json"
+tar -xzf "$rootfs_archive" -C "$bundle_b/rootfs"
+
+target/debug/a3s-oci oci-vm-multi-container-smoke \
+  --shim "$smoke_dir/a3s-oci-krun-shim" \
+  --vm-rootfs "$rootfs_dir/rootfs" \
+  --bundle-a "$bundle" \
+  --bundle-b "$bundle_b" \
+  --console "$rootfs_dir/oci-multi-container.log"
+```
+
+`a3s.oci.oci-vm-multi-container-smoke.v1` requires that starting, killing, and
+deleting A never changes B; recreating A advances generation 1 to 2; stale and
+cross-container replay requests fail; B then completes independently; and VM
+shutdown restores guest-runtime, endpoint, descriptor, shim, and worker
+inventories.
+
 Fault cleanup reuses the same signed shim and bundle but stops after each
 successful lifecycle boundary:
 
@@ -362,6 +407,12 @@ Matching retries reproduce the original result. Stale generations, reused
 operation IDs with different payloads, invalid isolation, and unsupported
 configuration fail before driver mutation.
 
+The shared Linux executor independently indexes every live `(container ID,
+generation)` pair, retains a highest-generation fence after delete, and gives
+each container a private runtime slot. The native Linux and macOS utility-VM
+multi-container gates keep two slots live concurrently and verify that every
+state transition and replay remains container-scoped.
+
 ### Linux executor boundary
 
 The current executor implements a reviewed bootstrap vertical slice:
@@ -401,9 +452,9 @@ boundary.
 
 | Host | Execution path | Retained evidence | Current readiness |
 | --- | --- | --- | --- |
-| Linux x86_64/aarch64 | Native Linux executor | Real rootful core lifecycle plus shutdown cleanup after create, start, and kill without delete; `/dev/kvm` absent and present-but-unusable | Default inventory `probe-only`; explicitly opened development instance `experimental` |
+| Linux x86_64/aarch64 | Native Linux executor | Real rootful core lifecycle, two-container generation and mutation isolation, plus shutdown cleanup after create, start, and kill without delete; `/dev/kvm` absent and present-but-unusable | Default inventory `probe-only`; explicitly opened development instance `experimental` |
 | Linux x86_64/aarch64 | libkrun + KVM utility VM | Device access, ioctl result, and KVM API version | `probe-only`; VM driver not implemented |
-| macOS arm64 | libkrun + HVF utility VM | Direct HVF VM create/destroy, checksum-pinned context lifecycle, authenticated static arm64 guest agent, fixed OCI core lifecycle, and no-delete cleanup after create, start, and kill | `probe-only`; complete enforcement and recovery pending |
+| macOS arm64 | libkrun + HVF utility VM | Direct HVF VM create/destroy, checksum-pinned context lifecycle, authenticated static arm64 guest agent, fixed and two-container OCI lifecycles, and no-delete cleanup after create, start, and kill | `probe-only`; complete enforcement and recovery pending |
 | Windows x86_64 | libkrun + WHPX utility VM | Partition, context, guest command, authenticated agent, and fixed OCI core lifecycle | `probe-only`; complete enforcement and recovery pending |
 
 Linux installation, feature inspection, and the native SDK path must work when

ROADMAP.md

@@ -270,7 +270,11 @@ then may HVF become `experimental`.
 
 ### R3 — Shared Linux Executor And Guest Agent
 
-- [ ] Multi-container guest registry with per-container generations.
+- [x] Multi-container guest registry with per-container generations, proven
+  with two distinct bundles, simultaneous create barriers, independent
+  start/kill/delete, exact replay isolation, generation-1 fencing after
+  generation-2 recreation, and complete cleanup through native Linux and the
+  macOS utility VM.
 - [x] Create a new UTS namespace and apply the configured hostname and
   domainname before the created barrier.
 - [x] Create a new mount namespace, make the inherited mount tree recursively