diff --git a/arch/x64/exceptions.cc b/arch/x64/exceptions.cc index 792691291..8b8418406 100644 --- a/arch/x64/exceptions.cc +++ b/arch/x64/exceptions.cc @@ -118,7 +118,12 @@ unsigned interrupt_descriptor_table::register_interrupt_handler( } } } - abort(); + // No free IDT vector. Vectors 0..31 are CPU exceptions and are never + // returned here, so 0 is an unambiguous "exhausted" sentinel. Returning it + // rather than aborting lets MSI-X allocation (msix_vector) fail gracefully + // and lets a device fall back to fewer queues instead of crashing the boot + // when many multiqueue devices exhaust the 224 usable vectors. + return 0; } void interrupt_descriptor_table::unregister_handler(unsigned vector) diff --git a/drivers/msi.cc b/drivers/msi.cc index 87366b957..ee9477aef 100644 --- a/drivers/msi.cc +++ b/drivers/msi.cc @@ -25,7 +25,11 @@ msix_vector::msix_vector(pci::function* dev) msix_vector::~msix_vector() { - idt.unregister_handler(_vector); + // _vector == 0 means IDT vector allocation failed (see the ctor); there is + // nothing to unregister in that case. + if (_vector) { + idt.unregister_handler(_vector); + } } pci::function* msix_vector::get_pci_function(void) @@ -198,7 +202,16 @@ std::vector interrupt_manager::request_vectors(unsigned num_vector auto num = std::min(num_vectors, num_entries); for (unsigned i = 0; i < num; ++i) { - results.push_back(new msix_vector(_dev)); + auto* vec = new msix_vector(_dev); + // A zero vector means the global IDT vector pool is exhausted. Stop + // here and return however many we did get; easy_register() treats a + // short result as failure so the caller can fall back to fewer queues + // rather than crash. + if (!vec->get_vector()) { + delete vec; + break; + } + results.push_back(vec); } return (results); diff --git a/drivers/virtio-blk.cc b/drivers/virtio-blk.cc index af3324d9a..132e380d5 100644 --- a/drivers/virtio-blk.cc +++ b/drivers/virtio-blk.cc @@ -137,6 +137,20 @@ blk::blk(virtio_device& virtio_dev) // base class did not set up. _num_queues = virtio_driver::_num_queues; + // Cap the number of queues we arm with an interrupt against the shared + // MSI-X vector budget. Each queue uses one vector, and the x86-64 IDT has + // only 224 usable vectors shared by every device; a high-vCPU guest with + // several multiqueue virtio devices can otherwise exhaust them at boot. + // Extra probed virtqueues beyond this cap are simply left unused - I/O is + // steered only to queues 0.._num_queues-1, all of which have a serviced + // interrupt. + unsigned granted = reserve_msix_vectors(_num_queues); + if (granted < (unsigned)_num_queues) { + virtio_i("virtio-blk: capping active queues from %d to %d (MSI-X budget)\n", + _num_queues, granted); + _num_queues = granted; + } + // Resize per-queue lock vector now that _num_queues is known. _queue_locks = std::vector(_num_queues); diff --git a/drivers/virtio-net.cc b/drivers/virtio-net.cc index b9e7d0426..607087e54 100644 --- a/drivers/virtio-net.cc +++ b/drivers/virtio-net.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -136,10 +137,13 @@ static int if_transmit(struct ifnet* ifp, struct mbuf* m_head) inline int net::xmit(struct mbuf* buff) { // - // We currently have only a single TX queue. Select a proper TXq here when - // we implement a multi-queue. + // Select a Tx queue by CPU id so that transmits from different CPUs use + // independent rings and do not serialize on one queue. With a single + // queue pair (_nqp == 1) this always selects queue 0, i.e. the original + // behaviour. // - return _txq.xmit(buff); + unsigned qidx = sched::cpu::current()->id % _nqp; + return _txqs[qidx]->xmit(buff); } inline int net::txq::xmit(mbuf* buff) @@ -186,9 +190,13 @@ static void if_getinfo(struct ifnet* ifp, struct if_data* out_data) void net::fill_stats(struct if_data* out_data) const { - // We currently support only a single Tx/Rx queue so no iteration so far - fill_qstats(_rxq, out_data); - fill_qstats(_txq, out_data); + // Aggregate the per-queue statistics across all Rx/Tx queue pairs. + for (auto& rxq : _rxqs) { + fill_qstats(*rxq, out_data); + } + for (auto& txq : _txqs) { + fill_qstats(*txq, out_data); + } } void net::fill_qstats(const struct rxq& rxq, struct if_data* out_data) const @@ -203,15 +211,16 @@ void net::fill_qstats(const struct rxq& rxq, struct if_data* out_data) const void net::fill_qstats(const struct txq& txq, struct if_data* out_data) const { - assert(!out_data->ifi_oerrors && !out_data->ifi_obytes && !out_data->ifi_opackets); + // Called once per Tx queue and accumulated, so every field must add rather + // than overwrite (with multiqueue there is more than one Tx queue). out_data->ifi_opackets += txq.stats.tx_packets; out_data->ifi_obytes += txq.stats.tx_bytes; out_data->ifi_oerrors += txq.stats.tx_err + txq.stats.tx_drops; - out_data->ifi_oworker_kicks = txq.stats.tx_worker_kicks; - out_data->ifi_oworker_wakeups = txq.stats.tx_worker_wakeups; - out_data->ifi_oworker_packets = txq.stats.tx_worker_packets; - out_data->ifi_okicks = txq.stats.tx_kicks; - out_data->ifi_oqueue_is_full = txq.stats.tx_hw_queue_is_full; + out_data->ifi_oworker_kicks += txq.stats.tx_worker_kicks; + out_data->ifi_oworker_wakeups += txq.stats.tx_worker_wakeups; + out_data->ifi_oworker_packets += txq.stats.tx_worker_packets; + out_data->ifi_okicks += txq.stats.tx_kicks; + out_data->ifi_oqueue_is_full += txq.stats.tx_hw_queue_is_full; out_data->ifi_owakeup_stats = txq.stats.tx_wakeup_stats; } @@ -220,7 +229,9 @@ bool net::ack_irq() auto isr = _dev.read_and_ack_isr(); if (isr) { - _rxq.vqueue->disable_interrupts(); + // Only used on the shared single-interrupt (MMIO/legacy) path, where + // there is a single queue pair; disable the Rx queue's interrupts. + _rxqs[0]->vqueue->disable_interrupts(); return true; } else { return false; @@ -239,18 +250,12 @@ void net::init() net::net(virtio_device& dev) : virtio_driver(dev), - _pre_init(this), - _rxq(get_virt_queue(0), [this] { this->receiver(); }), - _txq(this, get_virt_queue(1)) + _pre_init(this) { _driver_name = "virtio-net"; virtio_i("VIRTIO NET INSTANCE"); _id = _instance++; - sched::thread* poll_task = _rxq.poll_task.get(); - - poll_task->set_priority(sched::thread::priority_infinity); - // Please look at the section 5.1.6.1 of virtio specification for explanation if (_dev.is_modern()) { _hdr_size = sizeof(net_hdr_mrg_rxbuf); @@ -259,6 +264,62 @@ net::net(virtio_device& dev) _hdr_size = _mergeable_bufs ? sizeof(net_hdr_mrg_rxbuf) : sizeof(net_hdr); } + // + // Decide the number of Rx/Tx queue pairs to use. With VIRTIO_NET_F_MQ the + // device advertises max_virtqueue_pairs and lays out the virtqueues as + // rx[i]=2*i, tx[i]=2*i+1, ctrl=2*nqp. probe_virt_queues() (run from init() + // via _pre_init) has already created every virtqueue the device exposes, so + // _num_queues tells us how many actually exist. We use one queue pair per + // vCPU, bounded by what the device advertises, what was probed, and the + // shared MSI-X vector budget (2 vectors per pair). When F_MQ is absent this + // collapses to a single pair - identical to the original driver. + unsigned nqp = 1; + // Activating more than one queue pair requires the control virtqueue to + // send CTRL_MQ_VQ_PAIRS_SET, so multiqueue is gated on both features. + if (get_guest_feature_bit(VIRTIO_NET_F_MQ) && + get_guest_feature_bit(VIRTIO_NET_F_CTRL_VQ) && + _config.max_virtqueue_pairs > 1) { + nqp = std::min(_config.max_virtqueue_pairs, sched::cpus.size()); + // Number of pairs the probed virtqueues can support: N pairs need + // 2*N data queues plus one control queue. + unsigned probed_pairs = _num_queues > 1 ? (_num_queues - 1) / 2 : 1; + nqp = std::min(nqp, probed_pairs); + if (nqp < 1) { + nqp = 1; + } + // Cap against the shared MSI-X vector budget (2 vectors per pair). + unsigned vec_pairs = reserve_msix_vectors(2 * nqp) / 2; + if (vec_pairs < 1) { + vec_pairs = 1; + } + nqp = std::min(nqp, vec_pairs); + } + _nqp = nqp; + // Per the virtio spec the control virtqueue always sits at index + // 2*max_virtqueue_pairs (based on the device's advertised maximum), which + // is not the same as 2*_nqp when we activate fewer pairs than advertised. + if (get_guest_feature_bit(VIRTIO_NET_F_CTRL_VQ)) { + unsigned max_pairs = _config.max_virtqueue_pairs > 1 ? + _config.max_virtqueue_pairs : 1; + int ctrl_idx = 2 * max_pairs; + if ((unsigned)ctrl_idx < _num_queues) { + _ctrl_vq_idx = ctrl_idx; + } + } + + // Build the Rx/Tx queue pairs. Each Rx queue gets its own poll thread; each + // Tx queue its own xmitter. + for (unsigned i = 0; i < _nqp; i++) { + auto* rx_vq = get_virt_queue(2 * i); + auto* tx_vq = get_virt_queue(2 * i + 1); + assert(rx_vq && tx_vq); + _rxqs.push_back(std::unique_ptr( + new rxq(rx_vq, [this, i] { this->receiver(this->_rxqs[i].get()); }))); + _txqs.push_back(std::unique_ptr>( + aligned_new(this, tx_vq))); + _rxqs[i]->poll_task->set_priority(sched::thread::priority_infinity); + } + //initialize the BSD interface _if _ifn = if_alloc(IFT_ETHER); if (_ifn == NULL) { @@ -277,7 +338,7 @@ net::net(virtio_device& dev) _ifn->if_qflush = if_qflush; _ifn->if_init = if_init; _ifn->if_getinfo = if_getinfo; - IFQ_SET_MAXLEN(&_ifn->if_snd, _txq.vqueue->size()); + IFQ_SET_MAXLEN(&_ifn->if_snd, _txqs[0]->vqueue->size()); _ifn->if_capabilities = 0; @@ -297,30 +358,63 @@ net::net(virtio_device& dev) _ifn->if_capenable = _ifn->if_capabilities | IFCAP_HWSTATS; - //Start the polling thread before attaching it to the Rx interrupt - poll_task->start(); - _txq.start(); + //Start the polling threads before attaching them to the Rx interrupts + for (unsigned i = 0; i < _nqp; i++) { + _rxqs[i]->poll_task->start(); + _txqs[i]->start(); + } ether_ifattach(_ifn, _config.mac); + // Pin each Rx poll thread to a distinct CPU (round-robin) so that with + // multiqueue the per-queue receivers run in parallel across CPUs instead of + // contending for one. With a single queue pair this pins to CPU 0, which + // matches the priority_infinity single-thread behaviour of the original + // driver closely enough and keeps its Rx work on one CPU. + for (unsigned i = 0; i < _nqp; i++) { + sched::thread::pin(_rxqs[i]->poll_task.get(), sched::cpus[i % sched::cpus.size()]); + } + interrupt_factory int_factory; #if CONF_drivers_pci - int_factory.register_msi_bindings = [this,poll_task](interrupt_manager &msi) { - msi.easy_register({ - { 0, [&] { this->_rxq.vqueue->disable_interrupts(); }, poll_task }, - { 1, [&] { this->_txq.vqueue->disable_interrupts(); }, nullptr } - }); + // One MSI-X vector per data virtqueue. setup_queue() maps virtqueue index + // i -> MSI-X entry i (1:1), so the entry numbers are the virtqueue indices + // rx[i]=2*i and tx[i]=2*i+1. Each Rx vector wakes that queue's poll thread; + // Tx vectors just disable the queue's interrupts (Tx completion is reaped by + // the xmitter/gc path, as in the original single-queue driver). + int_factory.register_msi_bindings = [this](interrupt_manager &msi) { + std::vector bindings; + bindings.reserve(2 * _nqp); + for (unsigned i = 0; i < _nqp; i++) { + auto* rx = _rxqs[i].get(); + auto* tx = _txqs[i].get(); + sched::thread* poll = rx->poll_task.get(); + bindings.push_back({ (unsigned)(2 * i), + [rx] { rx->vqueue->disable_interrupts(); }, poll }); + bindings.push_back({ (unsigned)(2 * i + 1), + [tx] { tx->vqueue->disable_interrupts(); }, nullptr }); + } + if (!msi.easy_register(bindings)) { + // Vector allocation is bounded by reserve_msix_vectors() above, so + // this should not happen; if it does, fail loudly rather than run + // with queues whose completions are never serviced. + net_e("virtio-net: failed to register %d MSI-X vectors", 2 * _nqp); + abort(); + } }; - int_factory.create_pci_interrupt = [this,poll_task](pci::device &pci_dev) { + int_factory.create_pci_interrupt = [this](pci::device &pci_dev) { return new pci_interrupt( pci_dev, [=] { return this->ack_irq(); }, - [=] { poll_task->wake_with_irq_disabled(); }); + [=] { this->_rxqs[0]->poll_task->wake_with_irq_disabled(); }); }; #endif #if CONF_drivers_mmio + // The MMIO transport has a single shared interrupt line, so multiqueue Rx + // steering is not available there; _nqp is 1 on that path. + sched::thread* poll_task = _rxqs[0]->poll_task.get(); #ifdef __aarch64__ int_factory.create_spi_edge_interrupt = [this,poll_task]() { return new spi_interrupt( @@ -340,7 +434,18 @@ net::net(virtio_device& dev) _dev.register_interrupt(int_factory); - fill_rx_ring(); + // Tell the device how many queue pairs we will actually use. Must happen + // after the queues and their interrupts are set up and before we start + // feeding the Rx rings. + if (_nqp > 1 && _ctrl_vq_idx >= 0) { + if (!set_vq_pairs(_nqp)) { + net_w("virtio-net: CTRL_MQ_VQ_PAIRS_SET(%d) not acked by device", _nqp); + } + } + + for (unsigned i = 0; i < _nqp; i++) { + fill_rx_ring(_rxqs[i].get()); + } // Step 8 add_dev_status(VIRTIO_CONFIG_S_DRIVER_OK); @@ -384,6 +489,18 @@ void net::read_config() _host_tso4 = get_guest_feature_bit(VIRTIO_NET_F_HOST_TSO4); _guest_ufo = get_guest_feature_bit(VIRTIO_NET_F_GUEST_UFO); + // With VIRTIO_NET_F_MQ the device advertises how many Rx/Tx queue pairs it + // supports; read it so the constructor can decide how many to use. Default + // to a single pair otherwise. + _config.max_virtqueue_pairs = 1; + if (get_guest_feature_bit(VIRTIO_NET_F_MQ)) { + virtio_conf_read(offsetof(net_config, max_virtqueue_pairs), + &_config.max_virtqueue_pairs, + sizeof(_config.max_virtqueue_pairs)); + net_i("Multiqueue: device advertises %d queue pairs", + _config.max_virtqueue_pairs); + } + net_i("Features: %s=%d,%s=%d", "Status", _status, "TSO_ECN", _tso_ecn); net_i("Features: %s=%d,%s=%d", "Host TSO ECN", _host_tso_ecn, "CSUM", _csum); net_i("Features: %s=%d,%s=%d", "Guest_csum", _guest_csum, "guest tso4", _guest_tso4); @@ -459,9 +576,9 @@ bool net::bad_rx_csum(struct mbuf* m, struct net_hdr* hdr) return false; } -void net::receiver() +void net::receiver(struct rxq* rxq) { - vring* vq = _rxq.vqueue; + vring* vq = rxq->vqueue; std::vector packet; u64 rx_drops = 0, rx_packets = 0, csum_ok = 0; u64 csum_err = 0, rx_bytes = 0; @@ -473,8 +590,8 @@ void net::receiver() virtio_driver::wait_for_queue(vq, &vring::used_ring_not_empty); trace_virtio_net_rx_wake(); - _rxq.stats.rx_bh_wakeups++; - _rxq.update_wakeup_stats(rx_packets); + rxq->stats.rx_bh_wakeups++; + rxq->update_wakeup_stats(rx_packets); u32 len; int nbufs; @@ -490,7 +607,7 @@ void net::receiver() vq->get_buf_finalize(); if (vq->effective_avail_ring_count() >= refill_thresh) - fill_rx_ring(); + fill_rx_ring(rxq); // Bad packet/buffer - discard and continue to the next one if (len < _hdr_size + ETHER_HDR_LEN) { @@ -553,11 +670,11 @@ void net::receiver() } // Update the stats - _rxq.stats.rx_drops += rx_drops; - _rxq.stats.rx_packets += rx_packets; - _rxq.stats.rx_csum += csum_ok; - _rxq.stats.rx_csum_err += csum_err; - _rxq.stats.rx_bytes += rx_bytes; + rxq->stats.rx_drops += rx_drops; + rxq->stats.rx_packets += rx_packets; + rxq->stats.rx_csum += csum_ok; + rxq->stats.rx_csum_err += csum_err; + rxq->stats.rx_bytes += rx_bytes; } } @@ -619,11 +736,11 @@ void net::do_free_large_buffer(void* buffer) memory::free_phys_contiguous_aligned(buffer); } -void net::fill_rx_ring() +void net::fill_rx_ring(struct rxq* rxq) { trace_virtio_net_fill_rx_ring(_ifn->if_index); int added = 0; - vring* vq = _rxq.vqueue; + vring* vq = rxq->vqueue; int size_in_pages = _use_large_buffers ? LARGE_BUFFER_SIZE_IN_PAGES : 1; while (vq->avail_ring_not_empty()) { @@ -905,6 +1022,51 @@ void net::txq::gc() vqueue->get_buf_gc(); } +// Send VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET on the control virtqueue to tell the +// device how many Rx/Tx queue pairs the guest will use. The control queue is +// used only here at init and has no dedicated interrupt thread, so we submit +// the request and poll the used ring synchronously for the device's ACK. +bool net::set_vq_pairs(u16 pairs) +{ + vring* vq = get_virt_queue(_ctrl_vq_idx); + if (!vq) { + return false; + } + + struct { + net_ctrl_hdr hdr; + net_ctrl_mq mq; + net_ctrl_ack ack; + } cmd; + cmd.hdr.class_t = VIRTIO_NET_CTRL_MQ; + cmd.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; + cmd.mq.virtqueue_pairs = pairs; + cmd.ack = VIRTIO_NET_ERR; + + vq->init_sg(); + vq->add_out_sg(&cmd.hdr, sizeof(cmd.hdr)); + vq->add_out_sg(&cmd.mq, sizeof(cmd.mq)); + vq->add_in_sg(&cmd.ack, sizeof(cmd.ack)); + + if (!vq->add_buf(&cmd)) { + return false; + } + vq->kick(); + + // The control queue is quiescent at init; busy-wait briefly for the reply. + u32 len; + for (int spins = 0; spins < 100000; spins++) { + if (vq->used_ring_not_empty()) { + vq->get_buf_elem(&len); + vq->get_buf_finalize(); + break; + } + sched::thread::yield(); + } + + return cmd.ack == VIRTIO_NET_OK; +} + u64 net::get_driver_features() { auto base = virtio_driver::get_driver_features(); @@ -918,6 +1080,12 @@ u64 net::get_driver_features() | (1 << VIRTIO_NET_F_HOST_TSO4) \ | (1 << VIRTIO_NET_F_GUEST_ECN) | (1 << VIRTIO_NET_F_GUEST_UFO) + // Multiqueue and its control channel. VIRTIO_NET_F_MQ lets the + // device steer received flows across several Rx queues; + // VIRTIO_NET_F_CTRL_VQ is required to send the + // CTRL_MQ_VQ_PAIRS_SET command that activates them. + | (1 << VIRTIO_NET_F_CTRL_VQ) + | (1 << VIRTIO_NET_F_MQ) ); } diff --git a/drivers/virtio-net.hh b/drivers/virtio-net.hh index ccd421f16..2a11eb455 100644 --- a/drivers/virtio-net.hh +++ b/drivers/virtio-net.hh @@ -15,6 +15,7 @@ #include #include +#include #include "drivers/virtio.hh" #include "drivers/pci-device.hh" @@ -28,6 +29,14 @@ namespace virtio { class net : public virtio_driver { public: + // Forward declarations of the per-queue structs (defined in the private + // section below) so member functions can be declared with rxq*/txq*. + // Declared private to match the access of their definitions. +private: + struct rxq; + struct txq; +public: + // The feature bitmap for virtio net enum NetFeatures { VIRTIO_NET_F_CSUM = 0, /* Host handles pkts w/ partial csum */ @@ -220,8 +229,12 @@ public: void wait_for_queue(vring* queue); bool bad_rx_csum(struct mbuf* m, struct net_hdr* hdr); - void receiver(); - void fill_rx_ring(); + // receiver()/fill_rx_ring() operate on a specific Rx queue so that with + // multiqueue (VIRTIO_NET_F_MQ) each Rx queue is drained by its own poll + // thread on its own CPU, instead of all inbound traffic funnelling through + // one queue and one thread. + void receiver(struct rxq* rxq); + void fill_rx_ring(struct rxq* rxq); mbuf* packet_to_mbuf(const std::vector& iovec); static void free_buffer_and_refcnt(void* buffer, void* refcnt); static void free_large_buffer_and_refcnt(void* buffer, void* refcnt); @@ -312,7 +325,8 @@ private: } } _pre_init; - /* Single Rx queue object */ + /* An Rx queue object. With VIRTIO_NET_F_MQ there is one per queue pair, + * each drained by its own poll thread. */ struct rxq { rxq(vring* vq, std::function poll_func) : vqueue(vq), poll_task(sched::thread::make(poll_func, sched::thread::attr(). @@ -496,9 +510,20 @@ private: } } - /* We currently support only a single Rx+Tx queue */ - struct rxq _rxq; - struct txq _txq; + /* Rx+Tx queue pairs. Without VIRTIO_NET_F_MQ there is exactly one pair, + * preserving the original single-queue behaviour. txq is over-aligned + * (its percpu xmitter is cacheline aligned), so it is allocated with + * aligned_new and freed with the matching deleter. */ + std::vector> _rxqs; + std::vector>> _txqs; + /* Number of active queue pairs (1 when multiqueue is not negotiated). */ + unsigned _nqp = 1; + /* Control virtqueue index (2*_nqp) when VIRTIO_NET_F_CTRL_VQ is present. */ + int _ctrl_vq_idx = -1; + + /* Tell the device how many queue pairs the guest will use, per the + * VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET command. Returns true on device ACK. */ + bool set_vq_pairs(u16 pairs); //maintains the virtio instance number for multiple drives static int _instance; diff --git a/drivers/virtio.cc b/drivers/virtio.cc index c58b39148..3f2d8491d 100644 --- a/drivers/virtio.cc +++ b/drivers/virtio.cc @@ -6,6 +6,8 @@ */ #include +#include +#include #include "drivers/virtio.hh" #include "virtio-vring.hh" @@ -19,6 +21,38 @@ namespace virtio { int virtio_driver::_disk_idx = 0; +// Process-wide MSI-X vector budget shared by all virtio devices. The x86-64 +// IDT exposes 224 usable vectors (32..255); GSI/legacy IRQs, the APIC timer +// and a few IPIs also consume some, so we hand out at most a conservative +// fraction to per-queue MSI-X and leave headroom. Each multiqueue device caps +// its interrupt-bearing queue count against what remains here so that a +// high-vCPU guest with several such devices still boots. This is a soft cap: +// it only limits how many queues a device arms with a distinct interrupt, not +// how many virtqueues exist. +static std::atomic msix_vector_budget{192}; + +unsigned virtio_driver::reserve_msix_vectors(unsigned want) +{ + if (want == 0) { + return 0; + } + int remaining = msix_vector_budget.load(std::memory_order_relaxed); + while (true) { + if (remaining <= 0) { + // Pool empty: still allow a single vector so the device is usable + // (it will run single-queue / shared-interrupt). + return 1; + } + unsigned grant = std::min(want, (unsigned)remaining); + if (msix_vector_budget.compare_exchange_weak( + remaining, remaining - (int)grant, + std::memory_order_relaxed)) { + return grant; + } + // remaining reloaded by compare_exchange_weak; retry. + } +} + virtio_driver::virtio_driver(virtio_device& dev) : hw_driver() , _dev(dev) diff --git a/drivers/virtio.hh b/drivers/virtio.hh index 3e52348c7..f22a82078 100644 --- a/drivers/virtio.hh +++ b/drivers/virtio.hh @@ -104,6 +104,16 @@ public: size_t get_vring_alignment() { return _dev.get_vring_alignment();} + // Reserve up to `want` MSI-X vectors from a process-wide budget and return + // how many are actually available. The x86-64 IDT has only 224 usable + // vectors (32..255) shared by every device, so a high-vCPU guest with + // several multiqueue virtio devices can exhaust them and crash at boot. + // Drivers that scale their queue count with the vCPU count (virtio-blk, + // virtio-net multiqueue) call this to cap the number of queues they arm + // with an interrupt, so the total stays within the budget and every device + // still gets at least one vector. Returns 0 only if the pool is empty. + static unsigned reserve_msix_vectors(unsigned want); + protected: // Actual drivers should implement this on top of the basic ring features virtual u64 get_driver_features() { return 1 << VIRTIO_RING_F_INDIRECT_DESC | 1 << VIRTIO_RING_F_EVENT_IDX; }