Clean up some code, prepare to add rackscale system syscalls

Author: hunhoffe

kernel/src/arch/x86_64/rackscale/processops/request_core.rs

@@ -84,7 +84,7 @@ pub(crate) fn handle_request_core(hdr: &mut RPCHeader, payload: &mut [u8]) -> Re
         return construct_error_ret(hdr, payload, RPCError::NoFileDescForPid);
     }
     let local_pid = local_pid.unwrap();
-    info!("handle_request_core() start");
+    log::trace!("handle_request_core() start");
 
     // Parse request
     let core_req = match unsafe { decode::<RequestCoreReq>(payload) } {
@@ -166,7 +166,7 @@ pub(crate) fn request_core_work(rpc_client: &mut dyn RPCClient) -> () {
 
             log::info!("Client finished processing core work request");
         } else {
-            log::info!("Client received no work.")
+            log::trace!("Client received no work.")
         }
     }
 }
@@ -183,7 +183,9 @@ pub(crate) fn handle_request_core_work(
             .expect("failed to fetch core assignment deque for node");
         deque.pop_front()
     };
-    log::info!("handle_request_core_work() Found work={:?}", work);
+    if work.is_some() {
+        log::info!("handle_request_core_work() Found work={:?}", work);
+    }
     let result = RequestCoreWorkRes { work };
 
     // Populate output buffer & header

kernel/src/arch/x86_64/rackscale/syscalls.rs

@@ -9,7 +9,7 @@ use crate::fs::fd::FileDescriptor;
 use crate::memory::Frame;
 use crate::nrproc;
 use crate::process::{KernArcBuffer, UserSlice};
-use crate::syscalls::{FsDispatch, ProcessDispatch, SystemCallDispatch};
+use crate::syscalls::{FsDispatch, ProcessDispatch, SystemCallDispatch, SystemDispatch};
 
 use super::super::syscall::{Arch86SystemCall, Arch86SystemDispatch, Arch86VSpaceDispatch};
 use super::client::RPC_CLIENT;
@@ -31,9 +31,22 @@ pub(crate) struct Arch86LwkSystemCall {
 
 impl SystemCallDispatch<u64> for Arch86LwkSystemCall {}
 // Use x86 syscall processing for not yet implemented systems:
-impl Arch86SystemDispatch for Arch86LwkSystemCall {}
 impl Arch86VSpaceDispatch for Arch86LwkSystemCall {}
 
+impl SystemDispatch<u64> for Arch86LwkSystemCall {
+    fn get_hardware_threads(&self, vaddr_buf: u64, vaddr_buf_len: u64) -> KResult<(u64, u64)> {
+        self.local.get_hardware_threads(vaddr_buf, vaddr_buf_len)
+    }
+
+    fn get_stats(&self) -> KResult<(u64, u64)> {
+        self.local.get_stats()
+    }
+
+    fn get_core_id(&self) -> KResult<(u64, u64)> {
+        self.local.get_core_id()
+    }
+}
+
 impl FsDispatch<u64> for Arch86LwkSystemCall {
     fn open(&self, path: UserSlice, flags: FileFlags, modes: FileModes) -> KResult<(u64, u64)> {
         let pid = path.pid;

kernel/tests/s06_rackscale_tests.rs

@@ -488,7 +488,7 @@ fn rackscale_userspace_multicore_test(is_shmem: bool) {
 
 #[cfg(not(feature = "baremetal"))]
 #[test]
-fn s06_rackscale_shmem_request_core_remote() {
+fn s06_rackscale_shmem_request_core_remote_test() {
     use std::fs::remove_file;
     use std::sync::Arc;
     use std::thread::sleep;
@@ -501,16 +501,7 @@ fn s06_rackscale_shmem_request_core_remote() {
     let timeout = 30_000;
 
     // Create build for both controller and client
-    let controller_build = Arc::new(
-        BuildArgs::default()
-            .module("init")
-            .kernel_feature("shmem")
-            .kernel_feature("ethernet")
-            .kernel_feature("rackscale")
-            .release()
-            .build(),
-    );
-    let client_build = Arc::new(
+    let build = Arc::new(
         BuildArgs::default()
             .module("init")
             .user_feature("test-request-core-remote")
@@ -522,6 +513,7 @@ fn s06_rackscale_shmem_request_core_remote() {
     );
 
     // Run DCM and controller in separate thread
+    let controller_build = build.clone();
     let controller = std::thread::spawn(move || {
         let cmdline_controller = RunnerArgs::new_with_build("userspace-smp", &controller_build)
             .timeout(timeout)
@@ -538,7 +530,6 @@ fn s06_rackscale_shmem_request_core_remote() {
             let mut dcm = spawn_dcm(1, timeout)?;
             let mut p = spawn_nrk(&cmdline_controller)?;
 
-            //output += p.exp_string("Finished sending requests!")?.as_str();
             output += p.exp_eof()?.as_str();
 
             dcm.send_control('c')?;
@@ -549,7 +540,7 @@ fn s06_rackscale_shmem_request_core_remote() {
     });
 
     // Run client in separate thead. Wait a bit to make sure DCM and controller started
-    let client1_build = client_build.clone();
+    let client1_build = build.clone();
     let client = std::thread::spawn(move || {
         sleep(Duration::from_millis(5_000));
         let cmdline_client = RunnerArgs::new_with_build("userspace-smp", &client1_build)
@@ -562,6 +553,7 @@ fn s06_rackscale_shmem_request_core_remote() {
             .workers(3)
             .cores(2)
             .memory(4096)
+            .nobuild() // Use single build for all for consistency
             .use_vmxnet3();
 
         let mut output = String::new();
@@ -580,7 +572,7 @@ fn s06_rackscale_shmem_request_core_remote() {
     });
 
     // Run client in separate thead. Wait a bit to make sure DCM and controller started
-    let client2_build = client_build.clone();
+    let client2_build = build.clone();
     let client2 = std::thread::spawn(move || {
         sleep(Duration::from_millis(10_000));
         let cmdline_client = RunnerArgs::new_with_build("userspace-smp", &client2_build)
@@ -612,3 +604,130 @@ fn s06_rackscale_shmem_request_core_remote() {
 
     let _ignore = remove_file(SHMEM_PATH);
 }
+
+#[cfg(not(feature = "baremetal"))]
+#[test]
+fn s06_rackscale_spawn_test() {
+    use std::fs::remove_file;
+    use std::sync::Arc;
+    use std::thread::sleep;
+    use std::time::Duration;
+
+    // Setup ivshmem file
+    setup_shmem(SHMEM_PATH, SHMEM_SIZE);
+
+    setup_network(3);
+    let timeout = 30_000;
+
+    const TOTAL_CLIENT_CORES: usize = 3;
+
+    // Create build for both controller and client
+    let build = Arc::new(
+        BuildArgs::default()
+            .module("init")
+            .user_feature("test-scheduler-smp")
+            .kernel_feature("shmem")
+            .kernel_feature("ethernet")
+            .kernel_feature("rackscale")
+            .release()
+            .build(),
+    );
+
+    // Run DCM and controller in separate thread
+    let controller_build = build.clone();
+    let controller = std::thread::spawn(move || {
+        let cmdline_controller = RunnerArgs::new_with_build("userspace-smp", &controller_build)
+            .timeout(timeout)
+            .cmd("mode=controller transport=shmem")
+            .shmem_size(SHMEM_SIZE as usize)
+            .shmem_path(SHMEM_PATH)
+            .tap("tap0")
+            .no_network_setup()
+            .workers(3)
+            .use_vmxnet3();
+
+        let mut output = String::new();
+        let mut qemu_run = || -> Result<WaitStatus> {
+            let mut dcm = spawn_dcm(1, timeout)?;
+            let mut p = spawn_nrk(&cmdline_controller)?;
+
+            output += p.exp_eof()?.as_str();
+
+            dcm.send_control('c')?;
+            p.process.exit()
+        };
+
+        let _ignore = qemu_run();
+    });
+
+    sleep(Duration::from_millis(5_000));
+
+    // Run client in separate thead. Wait a bit to make sure DCM and controller started
+    let client1_build = build.clone();
+    let client = std::thread::spawn(move || {
+        let cmdline_client = RunnerArgs::new_with_build("userspace-smp", &client1_build)
+            .timeout(timeout)
+            .cmd("mode=client transport=shmem")
+            .shmem_size(SHMEM_SIZE as usize)
+            .shmem_path(SHMEM_PATH)
+            .tap("tap2")
+            .no_network_setup()
+            .workers(3)
+            .cores(2)
+            .memory(4096)
+            .use_vmxnet3();
+
+        let mut output = String::new();
+        let mut qemu_run = || -> Result<WaitStatus> {
+            let mut p = spawn_nrk(&cmdline_client)?;
+
+            // should get two requests for two cores
+            output += p
+                .exp_string("Client finished processing core work request")?
+                .as_str();
+            output += p
+                .exp_string("Client finished processing core work request")?
+                .as_str();
+
+            p.process.exit()
+        };
+        sleep(Duration::from_millis(10_000));
+
+        let _ignore = qemu_run();
+    });
+
+    // Run client in separate thead. Wait a bit to make sure DCM and controller started
+    let client2_build = build.clone();
+    let client2 = std::thread::spawn(move || {
+        let cmdline_client = RunnerArgs::new_with_build("userspace-smp", &client2_build)
+            .timeout(timeout)
+            .cmd("mode=client transport=shmem")
+            .shmem_size(SHMEM_SIZE as usize)
+            .shmem_path(SHMEM_PATH)
+            .tap("tap4")
+            .no_network_setup()
+            .workers(3)
+            .nobuild() // Use build from previous client for consistency
+            .use_vmxnet3();
+
+        let mut output = String::new();
+        let mut qemu_run = || -> Result<WaitStatus> {
+            let mut p = spawn_nrk(&cmdline_client)?;
+
+            for _i in 0..TOTAL_CLIENT_CORES {
+                let r = p.exp_regex(r#"init: Hello from core (\d+)"#)?;
+                output += r.0.as_str();
+                output += r.1.as_str();
+            }
+            p.process.kill(SIGTERM)
+        };
+
+        check_for_successful_exit(&cmdline_client, qemu_run(), output);
+    });
+
+    controller.join().unwrap();
+    client.join().unwrap();
+    client2.join().unwrap();
+
+    let _ignore = remove_file(SHMEM_PATH);
+}

usr/init/src/init.rs

@@ -150,34 +150,54 @@ fn request_core_remote_test() {
     let s = &vibrio::upcalls::PROCESS_SCHEDULER;
 
     let threads = vibrio::syscalls::System::threads().expect("Can't get system topology");
+    info!("Threads are: {:?}", threads);
 
-    // Only ask for more cores on the machine with only 1 core
-    if threads.len() == 1 {
-        let HW_THREAD_ID = 1;
-        match vibrio::syscalls::Process::request_core(
-            HW_THREAD_ID,
-            VAddr::from(vibrio::upcalls::upcall_while_enabled as *const fn() as u64),
-        ) {
-            Ok(_) => {
-                info!("request_core_remote_test OK");
-            }
-            Err(e) => {
-                error!("Can't spawn on {:?}: {:?}", HW_THREAD_ID, e);
+    /*
+    for thread in threads.iter() {
+        if thread.id != 0 {
+            let r = vibrio::syscalls::Process::request_core(
+                thread.id,
+                VAddr::from(vibrio::upcalls::upcall_while_enabled as *const fn() as u64),
+            );
+            match r {
+                Ok(ctoken) => {
+                    info!("Spawned core on {:?} <-> {}", ctoken, thread.id);
+                }
+                Err(_e) => {
+                    panic!("Failed to spawn to core {}", thread.id);
+                }
             }
         }
-    } else {
-        // Run scheduler on core 0
-        let scb: SchedulerControlBlock = SchedulerControlBlock::new(0);
-        loop {
-            s.run(&scb);
-        }
     }
+
+    for thread in threads {
+        s.spawn(
+            32 * 4096,
+            move |_| {
+                info!(
+                    "Hello from core {}",
+                    lineup::tls2::Environment::scheduler().core_id
+                );
+            },
+            ptr::null_mut(),
+            thread.id,
+            None,
+        );
+    }
+
+    // Run scheduler on core 0
+    let scb: SchedulerControlBlock = SchedulerControlBlock::new(0);
+    loop {
+        s.run(&scb);
+    }
+    */
 }
 
 fn scheduler_smp_test() {
     let s = &vibrio::upcalls::PROCESS_SCHEDULER;
 
     let threads = vibrio::syscalls::System::threads().expect("Can't get system topology");
+    info!("Threads are: {:?}", threads);
 
     for thread in threads.iter() {
         if thread.id != 0 {