create polling method for client to ask for core requests from

controller, added integration test for remote core request

Author: Erika Hunhoff

kernel/run.py

@@ -80,8 +80,7 @@ def get_network_config(workers):
 # General build arguments
 parser.add_argument("-v", "--verbose", action="store_true",
                     help="increase output verbosity")
-parser.add_argument("-n", "--norun", action="store_true",
-                    help="Only build, don't run")
+
 parser.add_argument("-r", "--release", action="store_true",
                     help="Do a release build.")
 parser.add_argument("--kfeatures", type=str, nargs='+', default=[],
@@ -97,6 +96,12 @@ def get_network_config(workers):
 parser.add_argument("--machine",
                     help='Which machine to run on (defaults to qemu)', required=False, default='qemu')
 
+parser_tasks_mut = parser.add_mutually_exclusive_group(required=False)
+parser_tasks_mut.add_argument("-n", "--norun", action="store_true", default=False,
+                    help="Only build, don't run")
+parser_tasks_mut.add_argument("-b", "--nobuild", action="store_true", default=False,
+                    help="Only run, don't build")
+
 
 # DCM Scheduler arguments
 parser.add_argument("--dcm-path",
@@ -812,11 +817,12 @@ def configure_dcm_scheduler(args):
         # Minimize python exception backtraces
         sys.excepthook = exception_handler
 
-    # Build
-    build_bootloader(args)
-    build_kernel(args)
-    build_user_libraries(args)
-    build_userspace(args)
+    if not args.nobuild:
+        # Build
+        build_bootloader(args)
+        build_kernel(args)
+        build_user_libraries(args)
+        build_userspace(args)
 
     # Deploy
     deploy(args)

kernel/src/arch/x86_64/irq.rs

@@ -503,6 +503,16 @@ unsafe fn timer_handler(_a: &ExceptionArguments) {
         nrproc::NrProcess::<Ring3Process>::synchronize(pid);
     }
 
+    // If this is a rackscale client, check for work from the controller
+    #[cfg(feature = "rackscale")]
+    if crate::CMDLINE
+        .get()
+        .map_or(false, |c| c.mode == crate::cmdline::Mode::Client)
+    {
+        use crate::arch::rackscale::client::client_get_work;
+        client_get_work();
+    }
+
     if super::process::has_executor() {
         // TODO(process-mgmt): Ensures that we still periodically
         // check and advance replicas even on cores that have a core.
@@ -529,7 +539,7 @@ unsafe fn timer_handler(_a: &ExceptionArguments) {
         r.resume()
     } else {
         // Go to scheduler instead
-        //warn!("got a timer on core {}", *crate::kcb::CORE_ID);
+        //warn!("got a timer on core {}", *crate::environment::CORE_ID);
         crate::scheduler::schedule()
     }
 }

kernel/src/arch/x86_64/rackscale/client.rs

@@ -11,6 +11,7 @@ use rpc::api::{RPCClient, RPCHandler, RegistrationHandler};
 use rpc::client::Client;
 use rpc::rpc::{ClientId, RPCError, RPCHeader};
 
+use crate::arch::rackscale::processops::request_core::request_core_work;
 use crate::cmdline::Transport;
 use crate::error::KError;
 use crate::fs::NrLock;
@@ -68,3 +69,8 @@ pub(crate) fn get_num_clients() -> u64 {
 pub(crate) fn get_local_client_id() -> u64 {
     (crate::CMDLINE.get().map_or(1, |c| c.machine_id) - 1) as u64
 }
+
+pub(crate) fn client_get_work() -> () {
+    let mut client = RPC_CLIENT.lock();
+    request_core_work(&mut **client);
+}

kernel/src/arch/x86_64/rackscale/controller.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
 use alloc::boxed::Box;
+use alloc::collections::vec_deque::VecDeque;
 use alloc::sync::Arc;
 use alloc::vec::Vec;
 use core::cell::Cell;
@@ -20,6 +21,7 @@ use rpc::server::Server;
 use crate::arch::debug::shutdown;
 use crate::arch::rackscale::client::get_num_clients;
 use crate::arch::rackscale::dcm::*;
+use crate::arch::rackscale::processops::request_core::RequestCoreReq;
 use crate::cmdline::Transport;
 use crate::error::KError;
 use crate::fs::{cnrfs, NrLock};
@@ -62,6 +64,20 @@ lazy_static! {
     };
 }
 
+// Keep track of unfulfilled core assignments
+lazy_static! {
+    pub(crate) static ref UNFULFILLED_CORE_ASSIGNMENTS: Arc<Mutex<Vec<Box<VecDeque<RequestCoreReq>>>>> = {
+        let mut core_assignments = Vec::try_with_capacity(get_num_clients() as usize)
+            .expect("Failed to create vector for core requests");
+        for i in 0..get_num_clients() {
+            // TODO: how to size vector appropriately? No try method for VecDeque
+            let mut client_core_assignments = VecDeque::with_capacity(3 as usize);
+            core_assignments.push(Box::new(client_core_assignments))
+        }
+        Arc::new(Mutex::new(core_assignments))
+    };
+}
+
 /// Test TCP RPC-based controller
 pub(crate) fn run() {
     // Create network interface and clock
@@ -192,7 +208,14 @@ fn register_rpcs(server: &mut Box<dyn RPCServer>) {
         )
         .unwrap();
     server
-        .register(KernelRpc::RequestCore as RPCType, &CORE_HANDLER)
+        .register(KernelRpc::RequestCore as RPCType, &REQUEST_CORE_HANDLER)
+        .unwrap();
+
+    server
+        .register(
+            KernelRpc::RequestWork as RPCType,
+            &REQUEST_CORE_WORK_HANDLER,
+        )
         .unwrap();
 }
 

kernel/src/arch/x86_64/rackscale/kernelrpc.rs

@@ -52,6 +52,8 @@ pub(crate) enum KernelRpc {
     ReleasePhysical = 14,
     /// Allocate a core for a process
     RequestCore = 15,
+    /// Request work (e.g., request cores) - used by client to ask controller for tasks
+    RequestWork = 16,
 }
 
 impl TryFrom<RPCType> for KernelRpc {
@@ -76,6 +78,7 @@ impl TryFrom<RPCType> for KernelRpc {
             13 => Ok(KernelRpc::AllocatePhysical),
             15 => Ok(KernelRpc::ReleasePhysical),
             15 => Ok(KernelRpc::RequestCore),
+            16 => Ok(KernelRpc::RequestWork),
             _ => Err(KError::InvalidRpcType),
         }
     }

kernel/src/arch/x86_64/rackscale/mod.rs

@@ -33,9 +33,13 @@ pub(crate) const READ_HANDLER: RPCHandler = fileops::rw::handle_read;
 pub(crate) const WRITE_HANDLER: RPCHandler = fileops::rw::handle_write;
 
 // Re-export handdlers: process operations
-pub(crate) const CORE_HANDLER: RPCHandler = processops::core::handle_request_core;
+pub(crate) const REQUEST_CORE_HANDLER: RPCHandler = processops::request_core::handle_request_core;
 pub(crate) const ALLOCATE_PHYSICAL_HANDLER: RPCHandler =
     processops::allocate_physical::handle_allocate_physical;
 pub(crate) const RELEASE_PHYSICAL_HANDLER: RPCHandler =
     processops::release_physical::handle_release_physical;
 pub(crate) const LOG_HANDLER: RPCHandler = processops::print::handle_log;
+
+// Client polls for work
+pub(crate) const REQUEST_CORE_WORK_HANDLER: RPCHandler =
+    processops::request_core::handle_request_core_work;

kernel/src/arch/x86_64/rackscale/processops/core.rs

@@ -2,6 +2,6 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
 pub mod allocate_physical;
-pub mod core;
 pub mod print;
 pub mod release_physical;
+pub mod request_core;