From 8763dadf80c49e6ec35e91e2009e2e6f92fc241e Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sat, 1 Aug 2026 21:12:59 -0400 Subject: [PATCH] drivers: bound MSI-X vector use so many-queue guests still boot The x86-64 IDT exposes only 224 usable interrupt vectors (32..255), and every device shares that pool. A guest with many vCPUs and several multiqueue virtio devices can request one MSI-X vector per queue and exhaust the pool during boot. Two things then went wrong: - register_interrupt_handler() called abort() when no IDT vector was free, crashing the boot instead of letting the device degrade. - request_vectors() pushed a msix_vector whose allocation had failed, and ~msix_vector() then unregistered vector 0. Fix both and add a soft cap so devices stay within budget in the first place: - register_interrupt_handler() returns 0 (an impossible real vector, since 0..31 are CPU exceptions) as an "exhausted" sentinel instead of aborting. msix_vector's destructor skips the unregister when its vector is 0, and request_vectors() stops at the first failed allocation and returns the short vector it did get; the existing easy_register() already treats a short result as failure, so a driver falls back to fewer queues rather than crashing. - Add virtio_driver::reserve_msix_vectors(): a process-wide soft budget that a multiqueue driver consults to cap how many queues it arms with a distinct interrupt, leaving headroom for other devices and always granting at least one. virtio-blk uses it to cap its active queue count; extra probed virtqueues are simply left unused. Signed-off-by: Greg Burd --- arch/x64/exceptions.cc | 7 ++++++- drivers/msi.cc | 17 +++++++++++++++-- drivers/virtio-blk.cc | 14 ++++++++++++++ drivers/virtio.cc | 34 ++++++++++++++++++++++++++++++++++ drivers/virtio.hh | 10 ++++++++++ 5 files changed, 79 insertions(+), 3 deletions(-) diff --git a/arch/x64/exceptions.cc b/arch/x64/exceptions.cc index 7926912915..8b84184068 100644 --- a/arch/x64/exceptions.cc +++ b/arch/x64/exceptions.cc @@ -118,7 +118,12 @@ unsigned interrupt_descriptor_table::register_interrupt_handler( } } } - abort(); + // No free IDT vector. Vectors 0..31 are CPU exceptions and are never + // returned here, so 0 is an unambiguous "exhausted" sentinel. Returning it + // rather than aborting lets MSI-X allocation (msix_vector) fail gracefully + // and lets a device fall back to fewer queues instead of crashing the boot + // when many multiqueue devices exhaust the 224 usable vectors. + return 0; } void interrupt_descriptor_table::unregister_handler(unsigned vector) diff --git a/drivers/msi.cc b/drivers/msi.cc index 87366b9576..ee9477aeff 100644 --- a/drivers/msi.cc +++ b/drivers/msi.cc @@ -25,7 +25,11 @@ msix_vector::msix_vector(pci::function* dev) msix_vector::~msix_vector() { - idt.unregister_handler(_vector); + // _vector == 0 means IDT vector allocation failed (see the ctor); there is + // nothing to unregister in that case. + if (_vector) { + idt.unregister_handler(_vector); + } } pci::function* msix_vector::get_pci_function(void) @@ -198,7 +202,16 @@ std::vector interrupt_manager::request_vectors(unsigned num_vector auto num = std::min(num_vectors, num_entries); for (unsigned i = 0; i < num; ++i) { - results.push_back(new msix_vector(_dev)); + auto* vec = new msix_vector(_dev); + // A zero vector means the global IDT vector pool is exhausted. Stop + // here and return however many we did get; easy_register() treats a + // short result as failure so the caller can fall back to fewer queues + // rather than crash. + if (!vec->get_vector()) { + delete vec; + break; + } + results.push_back(vec); } return (results); diff --git a/drivers/virtio-blk.cc b/drivers/virtio-blk.cc index af3324d9a2..0df1b40f8b 100644 --- a/drivers/virtio-blk.cc +++ b/drivers/virtio-blk.cc @@ -137,6 +137,20 @@ blk::blk(virtio_device& virtio_dev) // base class did not set up. _num_queues = virtio_driver::_num_queues; + // Cap the number of queues we arm with an interrupt against the shared + // MSI-X vector budget. Each queue uses one vector, and the x86-64 IDT has + // only 224 usable vectors shared by every device; a high-vCPU guest with + // several multiqueue virtio devices can otherwise exhaust them at boot. + // Extra probed virtqueues beyond this cap are simply left unused - I/O is + // directed only to queues 0.._num_queues-1, all of which have a serviced + // interrupt. + unsigned granted = reserve_msix_vectors(_num_queues); + if (granted < (unsigned)_num_queues) { + virtio_i("virtio-blk: capping active queues from %d to %d (MSI-X budget)\n", + _num_queues, granted); + _num_queues = granted; + } + // Resize per-queue lock vector now that _num_queues is known. _queue_locks = std::vector(_num_queues); diff --git a/drivers/virtio.cc b/drivers/virtio.cc index c58b39148d..3f2d8491d1 100644 --- a/drivers/virtio.cc +++ b/drivers/virtio.cc @@ -6,6 +6,8 @@ */ #include +#include +#include #include "drivers/virtio.hh" #include "virtio-vring.hh" @@ -19,6 +21,38 @@ namespace virtio { int virtio_driver::_disk_idx = 0; +// Process-wide MSI-X vector budget shared by all virtio devices. The x86-64 +// IDT exposes 224 usable vectors (32..255); GSI/legacy IRQs, the APIC timer +// and a few IPIs also consume some, so we hand out at most a conservative +// fraction to per-queue MSI-X and leave headroom. Each multiqueue device caps +// its interrupt-bearing queue count against what remains here so that a +// high-vCPU guest with several such devices still boots. This is a soft cap: +// it only limits how many queues a device arms with a distinct interrupt, not +// how many virtqueues exist. +static std::atomic msix_vector_budget{192}; + +unsigned virtio_driver::reserve_msix_vectors(unsigned want) +{ + if (want == 0) { + return 0; + } + int remaining = msix_vector_budget.load(std::memory_order_relaxed); + while (true) { + if (remaining <= 0) { + // Pool empty: still allow a single vector so the device is usable + // (it will run single-queue / shared-interrupt). + return 1; + } + unsigned grant = std::min(want, (unsigned)remaining); + if (msix_vector_budget.compare_exchange_weak( + remaining, remaining - (int)grant, + std::memory_order_relaxed)) { + return grant; + } + // remaining reloaded by compare_exchange_weak; retry. + } +} + virtio_driver::virtio_driver(virtio_device& dev) : hw_driver() , _dev(dev) diff --git a/drivers/virtio.hh b/drivers/virtio.hh index 3e52348c72..0e49e44c5c 100644 --- a/drivers/virtio.hh +++ b/drivers/virtio.hh @@ -104,6 +104,16 @@ public: size_t get_vring_alignment() { return _dev.get_vring_alignment();} + // Reserve up to `want` MSI-X vectors from a process-wide budget and return + // how many are actually available. The x86-64 IDT has only 224 usable + // vectors (32..255) shared by every device, so a high-vCPU guest with + // several multiqueue virtio devices can exhaust them and crash at boot. + // Drivers that scale their queue count with the vCPU count (for example + // virtio-blk multiqueue) call this to cap the number of queues they arm + // with an interrupt, so the total stays within the budget and every device + // still gets at least one vector. Returns 0 only if the pool is empty. + static unsigned reserve_msix_vectors(unsigned want); + protected: // Actual drivers should implement this on top of the basic ring features virtual u64 get_driver_features() { return 1 << VIRTIO_RING_F_INDIRECT_DESC | 1 << VIRTIO_RING_F_EVENT_IDX; }