From 1bbb1991635991066420622ff77b82da21ba5230 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sat, 11 Jul 2026 15:39:02 -0400 Subject: [PATCH 1/3] numa: discover NUMA topology from ACPI SRAT/SLIT OSv had no notion of NUMA: the scheduler and allocator treat memory as flat, which leaves performance on the table on multi-socket bare-metal (the large EC2/GCP metal instances that are a target for Postgres-over-ZFS). This is the first, discovery-only step. It adds a numa:: module that parses the ACPI SRAT (System Resource Affinity Table) and SLIT (System Locality Distance Information Table) at boot, right after acpi::init(), and exposes the topology: - numa::nr_nodes() / numa::available() - numa::node_of_cpu(cpu_id) -- resolved by correlating SRAT APIC ids with the APIC ids the MADT parse recorded on each sched::cpu - numa::distance(from, to) -- from SLIT (10 == local per ACPI convention), defaulting to 10 local / 20 remote when no SLIT is present - numa::memory_ranges() -- physical ranges tagged with their node It handles the SRAT CPU-affinity, x2APIC CPU-affinity and memory-affinity subtables, and only trusts SLIT if its locality count matches the node count SRAT reported. On a machine with no SRAT (the common single-node VM) it reports one flat node and available() == false; nothing changes behavior. This intentionally does NOT yet change allocation or scheduling; it only makes the topology available so a node-aware allocator, scheduler affinity, and mbind/get_mempolicy can build on it. Add tests/tst-numa.cc validating the invariants (>= 1 node, every CPU maps in range, distance diagonal == 10 and off-diagonal >= 10, memory ranges name valid nodes). Verified on OSv under KVM both without NUMA (reports 1 flat node) and with a QEMU 2-node -numa config (reports 2 nodes, 3 memory ranges, available). --- Makefile | 1 + core/numa.cc | 178 +++++++++++++++++++++++++++++++++++++++++ include/osv/numa.hh | 63 +++++++++++++++ loader.cc | 2 + modules/tests/Makefile | 1 + tests/tst-numa.cc | 55 +++++++++++++ 6 files changed, 300 insertions(+) create mode 100644 core/numa.cc create mode 100644 include/osv/numa.hh create mode 100644 tests/tst-numa.cc diff --git a/Makefile b/Makefile index 30a6571bb..4638be87c 100644 --- a/Makefile +++ b/Makefile @@ -1131,6 +1131,7 @@ endif objects += linux.o objects += core/commands.o objects += core/sched.o +objects += core/numa.o objects += core/mmio.o objects += core/kprintf.o ifeq ($(conf_tracepoints),1) diff --git a/core/numa.cc b/core/numa.cc new file mode 100644 index 000000000..38f762a8d --- /dev/null +++ b/core/numa.cc @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2026 Waldemar Kozaczuk + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + +#include +#include +#include + +#include +#include + +#include + +#if CONF_drivers_acpi +extern "C" { +#include "acpi.h" +} +#include +#endif + +namespace numa { + +static bool s_available = false; +static unsigned s_nr_nodes = 1; +// Map from a raw APIC id to the NUMA node it belongs to (from SRAT). +static std::unordered_map s_apic_to_node; +// Map from a sched cpu id to its NUMA node (resolved via APIC id). +static std::unordered_map s_cpu_to_node; +// SLIT distance matrix, row-major, s_nr_nodes x s_nr_nodes; empty if no SLIT. +static std::vector s_distances; +static std::vector s_mem_ranges; +static bool s_initialized = false; + +unsigned nr_nodes() { return s_nr_nodes; } +bool available() { return s_available; } +const std::vector& memory_ranges() { return s_mem_ranges; } + +unsigned node_of_cpu(unsigned cpu_id) +{ + auto it = s_cpu_to_node.find(cpu_id); + return it == s_cpu_to_node.end() ? 0 : it->second; +} + +unsigned distance(unsigned from, unsigned to) +{ + if (from == to) { + return 10; // ACPI convention: 10 == local. + } + if (!s_distances.empty() && from < s_nr_nodes && to < s_nr_nodes) { + return s_distances[from * s_nr_nodes + to]; + } + return 20; // Default remote distance when no SLIT is present. +} + +#if CONF_drivers_acpi +using boost::intrusive::get_parent_from_member; + +// Record the (apic id -> node) mapping and track the highest node seen. +static void record_cpu_affinity(uint32_t apic_id, unsigned node, unsigned& max_node) +{ + s_apic_to_node[apic_id] = node; + max_node = std::max(max_node, node); +} + +static void parse_srat() +{ + char sig[] = ACPI_SIG_SRAT; + ACPI_TABLE_HEADER* header; + if (AcpiGetTable(sig, 0, &header) != AE_OK) { + return; // No SRAT: leave the single-node fallback in place. + } + auto srat = get_parent_from_member(header, &ACPI_TABLE_SRAT::Header); + void* sub = srat + 1; + void* end = static_cast(srat) + srat->Header.Length; + unsigned max_node = 0; + + while (sub < end) { + auto s = static_cast(sub); + if (s->Length == 0) { + break; // Guard against a malformed zero-length subtable. + } + switch (s->Type) { + case ACPI_SRAT_TYPE_CPU_AFFINITY: { + auto a = get_parent_from_member(s, &ACPI_SRAT_CPU_AFFINITY::Header); + if (a->Flags & ACPI_SRAT_CPU_ENABLED) { + unsigned node = a->ProximityDomainLo | + (a->ProximityDomainHi[0] << 8) | + (a->ProximityDomainHi[1] << 16) | + (a->ProximityDomainHi[2] << 24); + record_cpu_affinity(a->ApicId, node, max_node); + } + break; + } + case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: { + auto a = get_parent_from_member(s, &ACPI_SRAT_X2APIC_CPU_AFFINITY::Header); + if (a->Flags & ACPI_SRAT_CPU_ENABLED) { + record_cpu_affinity(a->ApicId, a->ProximityDomain, max_node); + } + break; + } + case ACPI_SRAT_TYPE_MEMORY_AFFINITY: { + auto m = get_parent_from_member(s, &ACPI_SRAT_MEM_AFFINITY::Header); + if (m->Flags & ACPI_SRAT_MEM_ENABLED) { + s_mem_ranges.push_back(mem_range{ + m->BaseAddress, m->Length, m->ProximityDomain, + (m->Flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) != 0}); + max_node = std::max(max_node, (unsigned)m->ProximityDomain); + } + break; + } + default: + break; + } + sub = static_cast(sub) + s->Length; + } + + if (!s_apic_to_node.empty() || !s_mem_ranges.empty()) { + s_available = true; + s_nr_nodes = max_node + 1; + } +} + +static void parse_slit() +{ + char sig[] = ACPI_SIG_SLIT; + ACPI_TABLE_HEADER* header; + if (AcpiGetTable(sig, 0, &header) != AE_OK) { + return; + } + auto slit = get_parent_from_member(header, &ACPI_TABLE_SLIT::Header); + uint64_t n = slit->LocalityCount; + // Only trust SLIT if it agrees with the node count we saw in SRAT. + if (n == 0 || n != s_nr_nodes) { + return; + } + s_distances.assign(slit->Entry, slit->Entry + n * n); +} + +// Resolve the (apic id -> node) map into a (sched cpu id -> node) map. +static void resolve_cpus() +{ + for (auto* c : sched::cpus) { + auto it = s_apic_to_node.find(c->arch.apic_id); + if (it != s_apic_to_node.end()) { + s_cpu_to_node[c->id] = it->second; + } + } +} +#endif + +void init() +{ + if (s_initialized) { + return; + } + s_initialized = true; + +#if CONF_drivers_acpi + parse_srat(); + if (s_available) { + parse_slit(); + resolve_cpus(); + } +#endif + + if (s_available) { + debugf("NUMA: %u node(s), %zu CPU(s) mapped, %zu memory range(s)%s\n", + s_nr_nodes, s_cpu_to_node.size(), s_mem_ranges.size(), + s_distances.empty() ? ", no SLIT" : ""); + } else { + debugf("NUMA: no SRAT, assuming a single flat node\n"); + } +} + +} diff --git a/include/osv/numa.hh b/include/osv/numa.hh new file mode 100644 index 000000000..71ff61fe6 --- /dev/null +++ b/include/osv/numa.hh @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2026 Waldemar Kozaczuk + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + +#ifndef OSV_NUMA_HH +#define OSV_NUMA_HH + +#include +#include +#include + +// NUMA topology discovery. +// +// This module parses the ACPI SRAT (System Resource Affinity Table) and SLIT +// (System Locality Distance Information Table) to learn the machine's NUMA +// layout: which node each CPU belongs to, which physical memory ranges belong +// to each node, and the relative distances between nodes. +// +// This is discovery only: it does not change how memory is allocated or how +// threads are scheduled. It exposes the topology so later work (a node-aware +// allocator, scheduler affinity, mbind/get_mempolicy) can use it. On a machine +// with no SRAT (the common single-node virtual machine), the whole system is +// reported as one node (node 0) containing every CPU. + +namespace numa { + +// A contiguous physical memory range assigned to a NUMA node. +struct mem_range { + uint64_t base; + uint64_t length; + unsigned node; + bool hotpluggable; +}; + +// Discover the topology by parsing SRAT/SLIT. Safe to call once, after ACPI is +// initialized and the CPUs have been enumerated (so APIC ids are known). If no +// SRAT is present, initializes a single flat node. Idempotent. +void init(); + +// Number of NUMA nodes (>= 1). +unsigned nr_nodes(); + +// True if the topology came from a real SRAT (as opposed to the synthesized +// single-node fallback). +bool available(); + +// The node a CPU (by sched cpu id) belongs to, or 0 if unknown. +unsigned node_of_cpu(unsigned cpu_id); + +// The SLIT distance from node `from` to node `to`. Linux/ACPI convention: +// 10 == local (same node), higher == farther. Returns 10 for the diagonal and +// a default (10 local / 20 remote) when no SLIT is present. +unsigned distance(unsigned from, unsigned to); + +// The memory ranges discovered from SRAT (empty if none). +const std::vector& memory_ranges(); + +} + +#endif diff --git a/loader.cc b/loader.cc index 6c2b406e3..5e29a86e9 100644 --- a/loader.cc +++ b/loader.cc @@ -26,6 +26,7 @@ #endif /* __x86_64__ */ #include +#include #include #include "arch.hh" #include "arch-setup.hh" @@ -824,6 +825,7 @@ void main_cont(int loader_argc, char** loader_argv) #ifdef __x86_64__ #if CONF_drivers_acpi acpi::init(); + numa::init(); #endif #endif /* __x86_64__ */ diff --git a/modules/tests/Makefile b/modules/tests/Makefile index 391dbd9a8..aecff380f 100644 --- a/modules/tests/Makefile +++ b/modules/tests/Makefile @@ -120,6 +120,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-bsd-evh.so \ tst-fpu.so tst-preempt.so tst-tracepoint.so tst-hub.so \ misc-console.so misc-leak.so misc-readbench.so misc-mmap-anon-perf.so \ tst-mmap-file.so misc-mmap-big-file.so tst-mmap.so tst-huge.so \ + tst-numa.so \ tst-elf-permissions.so misc-mutex.so misc-sockets.so tst-condvar.so \ tst-queue-mpsc.so tst-af-local.so tst-pipe.so tst-yield.so \ misc-ctxsw.so tst-read.so tst-symlink.so tst-openat.so \ diff --git a/tests/tst-numa.cc b/tests/tst-numa.cc new file mode 100644 index 000000000..71dd9cf02 --- /dev/null +++ b/tests/tst-numa.cc @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2026 Waldemar Kozaczuk + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + +// Verifies NUMA topology discovery. Boot with QEMU's -numa options to see more +// than one node; without them (the default) the machine is reported as a single +// flat node. Built and run as part of the OSv test image. + +#include +#include + +#include +#include + +int main() +{ + std::cerr << "Running numa tests\n"; + + // There is always at least one node. + unsigned n = numa::nr_nodes(); + assert(n >= 1); + std::cerr << " nodes=" << n << " available=" << numa::available() << "\n"; + + // Every CPU maps to a node within range. + for (auto* c : sched::cpus) { + unsigned node = numa::node_of_cpu(c->id); + assert(node < n); + } + + // Distances: the diagonal is local (10); off-diagonal is >= local. + for (unsigned a = 0; a < n; a++) { + assert(numa::distance(a, a) == 10); + for (unsigned b = 0; b < n; b++) { + assert(numa::distance(a, b) >= 10); + } + } + + // Memory ranges (if any) all name a node within range. + for (auto& r : numa::memory_ranges()) { + assert(r.node < n); + assert(r.length > 0); + } + + // When SRAT is present, every CPU should have been mapped and the node + // count should match at least one memory range or cpu affinity. + if (numa::available()) { + std::cerr << " memory ranges=" << numa::memory_ranges().size() << "\n"; + } + + std::cerr << "numa tests PASSED\n"; + return 0; +} From 9bbee2e9ef1bd2bf360263e641c8b5c5729bea82 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sat, 11 Jul 2026 15:56:00 -0400 Subject: [PATCH 2/3] numa: report real topology from getcpu and get_mempolicy With NUMA topology now discovered (numa:: module), make the topology-query syscalls report it instead of a hardcoded single node: - sys_getcpu() now fills the node-out argument with numa::node_of_cpu() for the calling CPU rather than always 0. - get_mempolicy(): - MPOL_F_NODE returns the calling CPU's node. - MPOL_F_NODE | MPOL_F_ADDR returns the node backing the given address, resolved via a page-table walk (virt_to_phys_pt) and numa::node_of_phys(). - the allowed-nodes mask now sets a bit for every discovered node (not just node 0), and rejects a maxnode smaller than the node count with EINVAL. set_mempolicy() stays a no-op: the topology is known but the physical allocator is not yet node-aware, so a placement policy cannot be enforced. Its comment is updated to say so; enforcement will come with the node-aware allocator. Adds numa::node_of_phys() to map a physical address to its node. On a machine with no SRAT everything degrades to the previous single-node-0 behavior. Add tests/tst-numa-mempolicy.cc checking getcpu's node is in range and matches numa::node_of_cpu, get_mempolicy's MPOL_F_NODE and allowed-mask (bit count == node count, maxnode-too-small EINVAL), and the MPOL_F_ADDR path. Verified on OSv under KVM single-node and with a QEMU 2-node -numa config. Depends on the "numa: discover NUMA topology from ACPI SRAT/SLIT" change. --- core/numa.cc | 10 +++++ include/osv/numa.hh | 4 ++ linux.cc | 50 ++++++++++++++++------ modules/tests/Makefile | 2 +- tests/tst-numa-mempolicy.cc | 85 +++++++++++++++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 13 deletions(-) create mode 100644 tests/tst-numa-mempolicy.cc diff --git a/core/numa.cc b/core/numa.cc index 38f762a8d..1ffa4df63 100644 --- a/core/numa.cc +++ b/core/numa.cc @@ -44,6 +44,16 @@ unsigned node_of_cpu(unsigned cpu_id) return it == s_cpu_to_node.end() ? 0 : it->second; } +int node_of_phys(uint64_t phys) +{ + for (auto& r : s_mem_ranges) { + if (phys >= r.base && phys < r.base + r.length) { + return (int)r.node; + } + } + return -1; +} + unsigned distance(unsigned from, unsigned to) { if (from == to) { diff --git a/include/osv/numa.hh b/include/osv/numa.hh index 71ff61fe6..f764be750 100644 --- a/include/osv/numa.hh +++ b/include/osv/numa.hh @@ -50,6 +50,10 @@ bool available(); // The node a CPU (by sched cpu id) belongs to, or 0 if unknown. unsigned node_of_cpu(unsigned cpu_id); +// The node that owns a physical address, or -1 if it falls in no known range. +// Only meaningful when available() is true. +int node_of_phys(uint64_t phys); + // The SLIT distance from node `from` to node `to`. Linux/ACPI convention: // 10 == local (same node), higher == farther. Returns 10 for the diagonal and // a default (10 local / 20 remote) when no SLIT is present. diff --git a/linux.cc b/linux.cc index 58a018e9e..8338262f1 100644 --- a/linux.cc +++ b/linux.cc @@ -10,6 +10,8 @@ #include #include +#include +#include #include #include #include @@ -176,23 +178,44 @@ int futex(int *uaddr, int op, int val, const struct timespec *timeout, static long get_mempolicy(int *policy, unsigned long *nmask, unsigned long maxnode, void *addr, int flags) { - // As OSv has no support for NUMA nodes, we do here the minimum possible, - // which is basically to return the same policy (MPOL_DEFAULT) and list - // of nodes (just node 0) no matter if the caller asked for the default - // policy, the allowed policy, or the policy for a specific address. + // Report the topology discovered from ACPI SRAT/SLIT (numa::). When NUMA is + // not available (no SRAT, the common single-node VM) this degrades to the + // old single-node-0 behavior. We still apply MPOL_DEFAULT everywhere: the + // allocator is not yet node-aware, so we cannot honor a real policy, but we + // can at least answer topology queries truthfully. if ((flags & MPOL_F_NODE)) { - *policy = 0; // in this case, store a node id, not a policy + // Store a node id in *policy, not a policy value. + if (!policy) { + errno = EINVAL; + return -1; + } + if (flags & MPOL_F_ADDR) { + // The node backing the given address, if we can resolve it. + int node = -1; + if (numa::available() && addr) { + auto phys = mmu::virt_to_phys_pt(addr); + node = numa::node_of_phys(phys); + } + *policy = node < 0 ? 0 : node; + } else { + // The node the calling CPU runs on. + *policy = numa::node_of_cpu(sched::cpu::current()->id); + } return 0; } if (policy) { *policy = MPOL_DEFAULT; } if (nmask) { - if (maxnode < 1) { + if (maxnode < numa::nr_nodes()) { errno = EINVAL; return -1; } - nmask[0] |= 1; + // Set a bit for every node that exists. + for (unsigned n = 0; n < numa::nr_nodes(); n++) { + nmask[n / (8 * sizeof(unsigned long))] |= + 1UL << (n % (8 * sizeof(unsigned long))); + } } return 0; } @@ -202,9 +225,11 @@ static long get_mempolicy(int *policy, unsigned long *nmask, static long set_mempolicy(int policy, unsigned long *nmask, unsigned long maxnode) { - // OSv has very minimal support for NUMA - merely exposes - // all cpus as a single node0 and cannot really apply any meaningful policy - // Therefore we implement this as noop, ignore all arguments and return success + // The topology is now discovered (numa::), but the physical allocator is + // not yet node-aware, so we cannot actually enforce a placement policy. + // Accept the call as a no-op (ignoring the requested policy) rather than + // failing, so NUMA-aware programs run unmodified. Enforcement will come + // when the allocator learns about nodes. return 0; } #endif @@ -441,12 +466,13 @@ static long sys_getcwd(char *buf, unsigned long size) #define __NR_sys_getcpu __NR_getcpu static long sys_getcpu(unsigned int *cpu, unsigned int *node, void *tcache) { + auto *c = sched::cpu::current(); if (cpu) { - *cpu = sched::cpu::current()->id; + *cpu = c->id; } if (node) { - *node = 0; + *node = numa::node_of_cpu(c->id); } return 0; diff --git a/modules/tests/Makefile b/modules/tests/Makefile index aecff380f..d17ad24c7 100644 --- a/modules/tests/Makefile +++ b/modules/tests/Makefile @@ -120,7 +120,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-bsd-evh.so \ tst-fpu.so tst-preempt.so tst-tracepoint.so tst-hub.so \ misc-console.so misc-leak.so misc-readbench.so misc-mmap-anon-perf.so \ tst-mmap-file.so misc-mmap-big-file.so tst-mmap.so tst-huge.so \ - tst-numa.so \ + tst-numa.so tst-numa-mempolicy.so \ tst-elf-permissions.so misc-mutex.so misc-sockets.so tst-condvar.so \ tst-queue-mpsc.so tst-af-local.so tst-pipe.so tst-yield.so \ misc-ctxsw.so tst-read.so tst-symlink.so tst-openat.so \ diff --git a/tests/tst-numa-mempolicy.cc b/tests/tst-numa-mempolicy.cc new file mode 100644 index 000000000..b1795de6a --- /dev/null +++ b/tests/tst-numa-mempolicy.cc @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2026 Waldemar Kozaczuk + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + +// Verifies that getcpu and get_mempolicy report the NUMA topology discovered by +// the numa:: module (rather than a hardcoded single node). Boot with QEMU's +// -numa options to exercise more than one node. Built and run as part of the +// OSv test image. + +#include + +#include +#include +#include +#include +#include + +#include +#include + +#define MPOL_F_NODE (1 << 0) +#define MPOL_F_ADDR (1 << 1) + +static long get_mempolicy_(int *policy, unsigned long *nmask, + unsigned long maxnode, void *addr, int flags) +{ + return syscall(SYS_get_mempolicy, policy, nmask, maxnode, addr, flags); +} + +int main() +{ + std::cerr << "Running numa-mempolicy tests\n"; + unsigned nodes = numa::nr_nodes(); + + // getcpu (via sched_getcpu, which passes a node-out pointer) must report a + // node within range and consistent with numa::node_of_cpu. + unsigned cpu = 0, node = 0; + long r = syscall(SYS_getcpu, &cpu, &node, nullptr); + assert(r == 0); + assert(node < nodes); + assert(node == numa::node_of_cpu(cpu)); + + // get_mempolicy with MPOL_F_NODE reports the current CPU's node. + int cur_node = -1; + assert(get_mempolicy_(&cur_node, nullptr, 0, nullptr, MPOL_F_NODE) == 0); + assert(cur_node >= 0 && (unsigned)cur_node < nodes); + + // The allowed-nodes mask must have exactly `nodes` bits set at minimum, and + // maxnode too small must fail with EINVAL. + unsigned long mask[16] = {}; + int policy = -1; + assert(get_mempolicy_(&policy, mask, sizeof(mask) * 8, nullptr, 0) == 0); + int bits = 0; + for (unsigned i = 0; i < sizeof(mask) * 8; i++) { + if (mask[i / (8 * sizeof(unsigned long))] & + (1UL << (i % (8 * sizeof(unsigned long))))) { + bits++; + } + } + assert((unsigned)bits == nodes); + + if (nodes > 1) { + errno = 0; + assert(get_mempolicy_(&policy, mask, 1, nullptr, 0) == -1 && + errno == EINVAL); + } + + // MPOL_F_NODE|MPOL_F_ADDR reports the node backing a given address. On a + // real NUMA machine this must be a valid node; the value depends on + // placement so we only check the range. + void *buf = malloc(4096); + assert(buf); + *(volatile char *)buf = 1; // fault it in + int addr_node = -1; + assert(get_mempolicy_(&addr_node, nullptr, 0, buf, MPOL_F_NODE | MPOL_F_ADDR) == 0); + assert(addr_node >= 0 && (unsigned)addr_node < nodes); + free(buf); + + std::cerr << " nodes=" << nodes << " current_node=" << cur_node << "\n"; + std::cerr << "numa-mempolicy tests PASSED\n"; + return 0; +} From 16852d0fd53a37b525997e16f7cff821d84958ac Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sun, 12 Jul 2026 10:28:43 -0400 Subject: [PATCH 3/3] mm: add node-preferring page allocation (alloc_page_on_node) First incremental step of a node-aware allocator, on top of the NUMA topology discovery from the earlier commits. Adds memory::alloc_page_on_node(node), which prefers a page whose physical memory lies on the requested NUMA node, falling back to a normal allocation when that node has no free memory in the global page-range allocator (or when NUMA is not available / the node is out of range). Implementation layers on top of the existing allocator without restructuring it: page_range_allocator::alloc_page_from_node() walks the free lists, finds a range whose physical address resolves (via numa::node_of_phys()) to the requested node, and carves one page from it the same way alloc() does; if none is found it returns nullptr and alloc_page_on_node() falls back. This is deliberately a best-effort hint, not full per-node pools: OSv drains much of physical memory into per-CPU L1/L2 pools that are not yet node-partitioned, so once a node's global ranges are exhausted the allocator falls back. It gives correct placement while node-local global memory is available and never fails or misplaces a page. Full per-node L1/L2 pools are future work (roadmap B2.2-full); the scheduler affinity and mbind/set_mempolicy enforcement build on this hint. Add tests/tst-numa-alloc.cc: basic allocation and writability, out-of-range and negative node fall back cleanly, and (with QEMU -numa) node-0 allocations land on node 0 while other nodes fall back cleanly with no crash or misplacement. Passes on OSv under KVM single-node and with a 2-node -numa config; tst-mmap (the shared allocator path) unaffected. Depends on the NUMA topology-discovery and getcpu/get_mempolicy changes. --- core/mempool.cc | 72 ++++++++++++++++++++++++++++++++ include/osv/pagealloc.hh | 4 ++ modules/tests/Makefile | 2 +- tests/tst-numa-alloc.cc | 88 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 tests/tst-numa-alloc.cc diff --git a/core/mempool.cc b/core/mempool.cc index 7aced6829..e3212a054 100644 --- a/core/mempool.cc +++ b/core/mempool.cc @@ -23,6 +23,7 @@ #endif #include #include +#include #include #include #include @@ -597,6 +598,11 @@ class page_range_allocator { page_range* alloc(size_t size, bool contiguous = true); page_range* alloc_aligned(size_t size, size_t offset, size_t alignment, bool fill = false); + // Carve a single page from a free range that lies on NUMA node `node`, + // returning it, or nullptr if no free memory on that node is available. + // Used by the node-preferring allocator; the caller falls back to a normal + // allocation when this returns nullptr. + page_range* alloc_page_from_node(int node); void free(page_range* pr); void initial_add(page_range* pr); @@ -796,6 +802,38 @@ page_range* page_range_allocator::alloc(size_t size, bool contiguous) return ≺ } +page_range* page_range_allocator::alloc_page_from_node(int node) +{ + // Find a free range whose memory lies on the requested NUMA node, then + // carve a single page from its front (mirroring alloc()). This is a + // best-effort preference: a linear scan over the free lists, returning the + // first range on the node. Returns nullptr if none is found, so the caller + // can fall back to a node-agnostic allocation. + page_range* found = nullptr; + for_each(0, [&] (page_range& pr) { + auto phys = mmu::virt_to_phys(static_cast(&pr)); + if (numa::node_of_phys(phys) == node) { + found = ≺ + return false; // stop iterating + } + return true; + }); + if (!found) { + return nullptr; + } + + auto& pr = *found; + remove(pr); + if (pr.size > page_size) { + auto& np = *new (static_cast(&pr) + page_size) + page_range(pr.size - page_size); + insert(np); + pr.size = page_size; + } + set_bits(pr, false); + return ≺ +} + page_range* page_range_allocator::alloc_aligned(size_t size, size_t offset, size_t alignment, bool fill) { @@ -1789,6 +1827,40 @@ void* alloc_page() return p; } +// Allocate one page, preferring memory local to NUMA node `node`. Falls back +// to a node-agnostic allocation when the node has no free memory local to it +// in the global page-range allocator (or NUMA is not available). +// +// This is a best-effort node-preferring hint that the scheduler and +// mbind/set_mempolicy can build on. It only sees memory still held by the +// global page-range allocator: OSv drains much of physical memory into per-CPU +// L1/L2 pools that are not (yet) node-partitioned, so once a node's global +// ranges are exhausted this falls back. Full per-node pools are future work +// (roadmap B2.2-full); this step gives correct placement while node-local +// global memory is available and never fails or misplaces. +void* alloc_page_on_node(int node) +{ + if (node < 0 || !numa::available() || (unsigned)node >= numa::nr_nodes()) { + return alloc_page(); + } + void* ret = nullptr; + WITH_LOCK(free_page_ranges_lock) { + page_range* pr = free_page_ranges.alloc_page_from_node(node); + if (pr) { + on_alloc(page_size); + ret = static_cast(pr); + } + } + if (!ret) { + // No free memory on that node: fall back rather than fail. + return alloc_page(); + } +#if CONF_memory_tracker + tracker_remember(ret, page_size); +#endif + return ret; +} + static inline void untracked_free_page(void *v) { trace_memory_page_free(v); diff --git a/include/osv/pagealloc.hh b/include/osv/pagealloc.hh index bbc6e9df1..3978ddba1 100644 --- a/include/osv/pagealloc.hh +++ b/include/osv/pagealloc.hh @@ -13,6 +13,10 @@ namespace memory { void* alloc_page(); +// Allocate one page preferring memory local to NUMA node `node`, falling back +// to a node-agnostic allocation when the node has no free memory or NUMA is +// unavailable. +void* alloc_page_on_node(int node); void free_page(void* page); void* alloc_huge_page(size_t bytes); void free_huge_page(void *page, size_t bytes); diff --git a/modules/tests/Makefile b/modules/tests/Makefile index d17ad24c7..c3886d0b9 100644 --- a/modules/tests/Makefile +++ b/modules/tests/Makefile @@ -120,7 +120,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-bsd-evh.so \ tst-fpu.so tst-preempt.so tst-tracepoint.so tst-hub.so \ misc-console.so misc-leak.so misc-readbench.so misc-mmap-anon-perf.so \ tst-mmap-file.so misc-mmap-big-file.so tst-mmap.so tst-huge.so \ - tst-numa.so tst-numa-mempolicy.so \ + tst-numa.so tst-numa-mempolicy.so tst-numa-alloc.so \ tst-elf-permissions.so misc-mutex.so misc-sockets.so tst-condvar.so \ tst-queue-mpsc.so tst-af-local.so tst-pipe.so tst-yield.so \ misc-ctxsw.so tst-read.so tst-symlink.so tst-openat.so \ diff --git a/tests/tst-numa-alloc.cc b/tests/tst-numa-alloc.cc new file mode 100644 index 000000000..6a61f107e --- /dev/null +++ b/tests/tst-numa-alloc.cc @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2026 Waldemar Kozaczuk + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + +// Verifies memory::alloc_page_on_node(): pages requested for a node land on +// that node (when NUMA is available), and it falls back cleanly otherwise. +// Boot with QEMU -numa to exercise multiple nodes. Built and run on the OSv +// test image. + +#include +#include +#include + +#include +#include +#include + +int main() +{ + std::cerr << "Running numa-alloc tests\n"; + unsigned nodes = numa::nr_nodes(); + std::cerr << " nodes=" << nodes << " available=" << numa::available() << "\n"; + + // Basic: allocating on node 0 always returns a usable page. + void *p = memory::alloc_page_on_node(0); + assert(p != nullptr); + // Writable. + *(volatile char *)p = 0x5a; + assert(*(volatile char *)p == 0x5a); + memory::free_page(p); + + // Out-of-range / negative node falls back to a normal allocation, no crash. + p = memory::alloc_page_on_node(-1); + assert(p != nullptr); + memory::free_page(p); + p = memory::alloc_page_on_node(nodes + 100); + assert(p != nullptr); + memory::free_page(p); + + // When NUMA is available with real memory ranges, a page requested for a + // node should actually be on that node (until the node runs out, when it + // falls back). Allocate a handful per node and check placement. + // When NUMA is available, a page requested for a node is served from that + // node's free memory when the global page-range allocator still holds free + // ranges there. OSv drains most physical memory into per-CPU pools at + // boot, so higher nodes may have no free ranges left in the global pool by + // the time we run; in that case alloc_page_on_node() falls back cleanly. + // The invariant we check: every returned page is valid and writable, its + // resolved node is either the requested one or a valid fallback, and at + // least one node actually places on-node (node 0 always has free ranges). + if (numa::available()) { + int total_on_node = 0; + for (unsigned n = 0; n < nodes; n++) { + std::vector pages; + int on_node = 0, off_node = 0; + for (int i = 0; i < 32; i++) { + void *q = memory::alloc_page_on_node(n); + assert(q != nullptr); + *(volatile char *)q = (char)i; // writable + pages.push_back(q); + auto phys = mmu::virt_to_phys(q); + int got = numa::node_of_phys(phys); + // The resolved node is either the requested node or a real node + // (fallback), never garbage. + assert(got == -1 || (got >= 0 && (unsigned)got < nodes)); + if (got == (int)n) { + on_node++; + } else { + off_node++; // fell back (node had no free ranges left) + } + } + total_on_node += on_node; + std::cerr << " node " << n << ": " << on_node << " on-node, " + << off_node << " fell back\n"; + for (void *q : pages) { + memory::free_page(q); + } + } + // At least one node must actually place on-node (node 0's free ranges). + assert(total_on_node > 0); + } + + std::cerr << "numa-alloc tests PASSED\n"; + return 0; +}