server: build Tokio runtime using oxide-tokio-rt (#913)

As described in [RFD 579], we are now recommending that the
[`oxide-tokio-rt`] crate be used to construct the Tokio runtime in all
production software that uses Tokio. This crate provides common
functions for constructing Tokio runtimes with the recommended
configurations.

In particular, it currently does the following:

- On illumos, configures the runtime to emit DTrace probes, using
  [`tokio-dtrace`].
- Disables Tokio's LIFO slot optimization. This feature is intended to
  improve message-passing latency, but because tasks in the LIFO slot
  do not currently participate in work-stealing, it can result in
  extreme latency spikes in some cases (see omicron#8334 for a worked
  example).

Thus, this commit changes `propolis-server` to construct its Tokio
runtimes using `oxide-tokio-rt` so that these common recommended
configurations are included. This requires enabling Tokio's unstable
features in the workspace.

[RFD 579]: https://rfd.shared.oxide.computer/rfd/0579
[`oxide-tokio-rt`]: https://github.com/oxidecomputer/oxide-tokio-rt
[`tokio-dtrace`]: https://github.com/oxidecomputer/tokio-dtrace

Author: hawkw

.cargo/config.toml

@@ -5,3 +5,11 @@ xtask = "run --package xtask --quiet --"
 # Currently required by Falcon due to
 # https://github.com/rust-lang/cargo/issues/3946#issuecomment-973132993
 CARGO_WORKSPACE_DIR = { value = "", relative = true }
+
+[build]
+# Tokio's unstable features are required by `tokio-dtrace` probes, and for
+# disabling the LIFO slot optimization.
+#
+# See here for details:
+# https://github.com/oxidecomputer/oxide-tokio-rt/blob/main/README.md#enabling-tokio_unstable-features
+rustflags = ["--cfg", "tokio_unstable"]

Cargo.lock

@@ -132,6 +132,7 @@ mockall = "0.12"
 newtype_derive = "0.1.6"
 newtype-uuid = { version = "1.0.1", features = [ "v4" ] }
 owo-colors = "4"
+oxide-tokio-rt = "0.1.2"
 pin-project-lite = "0.2.13"
 proc-macro2 = "1.0"
 proc-macro-error = "1"

Cargo.toml

@@ -39,6 +39,7 @@ kstat-rs.workspace = true
 lazy_static.workspace = true
 nexus-client.workspace = true
 omicron-common.workspace = true
+oxide-tokio-rt.workspace = true
 oximeter-instruments.workspace = true
 oximeter-producer.workspace = true
 oximeter.workspace = true

bin/propolis-server/Cargo.toml

@@ -263,14 +263,14 @@ impl<'a> VmEnsureNotStarted<'a> {
         let result: InitResult = tokio::task::spawn_blocking(move || {
             // Create the runtime that will host tasks created by
             // VMM components (e.g. block device runtime tasks).
-            let vmm_rt = tokio::runtime::Builder::new_multi_thread()
-                .thread_name("tokio-rt-vmm")
-                .worker_threads(usize::max(
+            let vmm_rt = {
+                let mut builder = tokio::runtime::Builder::new_multi_thread();
+                builder.thread_name("tokio-rt-vmm").worker_threads(usize::max(
                     VMM_MIN_RT_THREADS,
                     VMM_BASE_RT_THREADS + spec.board.cpus as usize,
-                ))
-                .enable_all()
-                .build()?;
+                ));
+                oxide_tokio_rt::build(&mut builder)?
+            };
 
             let init_result = vmm_rt
                 .block_on(async move {

bin/propolis-server/src/lib/vm/ensure.rs

@@ -160,11 +160,11 @@ fn run_server(
     // Spawn the runtime for handling API processing
     // If/when a VM instance is created, a separate runtime for handling device
     // emulation and other VM-related work will be spawned.
-    let api_runtime = tokio::runtime::Builder::new_multi_thread()
-        .worker_threads(API_RT_THREADS)
-        .enable_all()
-        .thread_name("tokio-rt-api")
-        .build()?;
+    let api_runtime = {
+        let mut builder = tokio::runtime::Builder::new_multi_thread();
+        builder.worker_threads(API_RT_THREADS).thread_name("tokio-rt-api");
+        oxide_tokio_rt::build(&mut builder)?
+    };
     let _guard = api_runtime.enter();
 
     // Start TCP listener for VNC, if requested

bin/propolis-server/src/main.rs

@@ -24,6 +24,7 @@ libc.workspace = true
 toml.workspace = true
 tokio = { workspace = true, features = ["io-util", "rt-multi-thread"] }
 serde = { workspace = true, features = ["derive"] }
+oxide-tokio-rt.workspace = true
 propolis.workspace = true
 propolis_types.workspace = true
 crucible-client-types = { workspace = true, optional = true }

bin/propolis-standalone/Cargo.toml

@@ -1490,11 +1490,11 @@ fn main() -> anyhow::Result<ExitCode> {
     // since we'll block in main when we call `Instance::wait_for_state`
     let rt_threads =
         MIN_RT_THREADS.max(BASE_RT_THREADS + config.main.cpus as usize);
-    let rt = tokio::runtime::Builder::new_multi_thread()
-        .worker_threads(rt_threads)
-        .enable_all()
-        .thread_name("vmm-tokio")
-        .build()?;
+    let rt = {
+        let mut builder = tokio::runtime::Builder::new_multi_thread();
+        builder.worker_threads(rt_threads).thread_name("vmm-tokio");
+        oxide_tokio_rt::build(&mut builder)?
+    };
     let _rt_guard = rt.enter();
 
     // Create the VM afresh or restore it from a snapshot