test(apf): add functional and perf tests

Add integration and performance tests for APF to compare the perf impact

Signed-off-by: Jack Thomson <jackabt@amazon.com>

Author: JackThomson2

src/firecracker/examples/uffd/uffd_utils.rs

@@ -885,7 +885,6 @@ impl Runtime {
                                 "received bogus offset from firecracker"
                             );
 
-                            // Handle one of FaultRequest page faults
                             pf_vcpu_event_dispatch(
                                 &mut self.handler,
                                 fault_request.offset as usize,

src/vmm/src/builder.rs

@@ -769,7 +769,8 @@ pub fn build_microvm_from_snapshot(
         use vmm_sys_util::sock_ctrl_msg::ScmSocket;
         if let Some(uff_socket) = uffd_socket {
             let broker = UffdMessageBroker::new(uff_socket);
-            let (contexts, done) = if apf_supported {
+            let apf_exitless = apf_supported && std::env::var("FC_APF_NO_EXITLESS").is_err();
+            let (contexts, done) = if apf_exitless {
                 let mut contexts = Vec::new();
                 let mut handler_fds = Vec::new();
                 for &vcpu_fd in &vcpu_fds {

tests/framework/microvm.py

@@ -1092,6 +1092,7 @@ def restore_from_snapshot(
         rename_interfaces: dict = None,
         *,
         uffd_handler_name: str = None,
+        apf: bool = True,
     ):
         """Restore a snapshot"""
 
@@ -1102,6 +1103,7 @@ def restore_from_snapshot(
                 self,
                 uffd_handler(uffd_handler_name, binary_dir=self.fc_binary_path.parent),
                 jailed_snapshot,
+                apf=apf,
             )
 
         jailed_mem = Path("/") / jailed_snapshot.mem.name
@@ -1308,12 +1310,15 @@ def build(self, kernel=None, rootfs=None, **kwargs):
             vm.ssh_key = ssh_key
         return vm
 
-    def build_from_snapshot(self, snapshot: Snapshot, uffd_handler_name=None):
+    def build_from_snapshot(self, snapshot: Snapshot, uffd_handler_name=None, apf=True):
         """Build a microvm from a snapshot"""
         vm = self.build()
         vm.spawn()
         vm.restore_from_snapshot(
-            snapshot, resume=True, uffd_handler_name=uffd_handler_name
+            snapshot,
+            resume=True,
+            uffd_handler_name=uffd_handler_name,
+            apf=apf,
         )
         return vm
 
@@ -1323,6 +1328,7 @@ def build_n_from_snapshot(
         nr_vms,
         *,
         uffd_handler_name=None,
+        apf=True,
         incremental=False,
         use_snapshot_editor=True,
         no_netns_reuse=False,
@@ -1342,7 +1348,10 @@ def build_n_from_snapshot(
             microvm.spawn()
 
             snapshot_copy = microvm.restore_from_snapshot(
-                current_snapshot, resume=True, uffd_handler_name=uffd_handler_name
+                current_snapshot,
+                resume=True,
+                uffd_handler_name=uffd_handler_name,
+                apf=apf,
             )
 
             yield microvm

tests/framework/utils_uffd.py

@@ -19,7 +19,13 @@ class UffdHandler:
     """Describe the UFFD page fault handler process."""
 
     def __init__(
-        self, name, socket_path, snapshot: "Snapshot", chroot_path, log_file_name
+        self,
+        name,
+        socket_path,
+        snapshot: "Snapshot",
+        chroot_path,
+        log_file_name,
+        apf=True,
     ):
         """Instantiate the handler process with arguments."""
         self._proc = None
@@ -28,6 +34,7 @@ def __init__(
         self.snapshot = snapshot
         self._chroot = chroot_path
         self._log_file = log_file_name
+        self._apf = apf
 
     def spawn(self, uid, gid):
         """Spawn handler process using arguments provided."""
@@ -42,8 +49,9 @@ def spawn(self, uid, gid):
                     f"/{self._handler_name}",
                     self.socket_path,
                     self.snapshot.mem.name,
-                    APF_SOCKET_PATH,
                 ]
+                if self._apf:
+                    args.append(APF_SOCKET_PATH)
                 self._proc = subprocess.Popen(
                     args, stdout=logfile, stderr=subprocess.STDOUT
                 )
@@ -99,15 +107,20 @@ def __del__(self):
             self.kill()
 
 
-def spawn_pf_handler(vm, handler_path, jailed_snapshot):
+def spawn_pf_handler(vm, handler_path, jailed_snapshot, apf=True):
     """Spawn page fault handler process."""
     # Copy snapshot memory file into chroot of microVM.
     # Copy the valid page fault binary into chroot of microVM.
     jailed_handler = vm.create_jailed_resource(handler_path)
     handler_name = os.path.basename(jailed_handler)
 
     uffd_handler = UffdHandler(
-        handler_name, SOCKET_PATH, jailed_snapshot, vm.chroot(), "uffd.log"
+        handler_name,
+        SOCKET_PATH,
+        jailed_snapshot,
+        vm.chroot(),
+        "uffd.log",
+        apf=apf,
     )
     uffd_handler.spawn(vm.jailer.uid, vm.jailer.gid)