numa: report real topology from getcpu and get_mempolicy

Makefile

@@ -1131,6 +1131,7 @@ endif
 objects += linux.o
 objects += core/commands.o
 objects += core/sched.o
+objects += core/numa.o
 objects += core/mmio.o
 objects += core/kprintf.o
 ifeq ($(conf_tracepoints),1)

core/numa.cc

@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2026 Waldemar Kozaczuk
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include <osv/numa.hh>
+#include <osv/sched.hh>
+#include <osv/debug.hh>
+
+#include <unordered_map>
+#include <algorithm>
+
+#include <osv/drivers_config.h>
+
+#if CONF_drivers_acpi
+extern "C" {
+#include "acpi.h"
+}
+#include <boost/intrusive/parent_from_member.hpp>
+#endif
+
+namespace numa {
+
+static bool s_available = false;
+static unsigned s_nr_nodes = 1;
+// Map from a raw APIC id to the NUMA node it belongs to (from SRAT).
+static std::unordered_map<uint32_t, unsigned> s_apic_to_node;
+// Map from a sched cpu id to its NUMA node (resolved via APIC id).
+static std::unordered_map<unsigned, unsigned> s_cpu_to_node;
+// SLIT distance matrix, row-major, s_nr_nodes x s_nr_nodes; empty if no SLIT.
+static std::vector<uint8_t> s_distances;
+static std::vector<mem_range> s_mem_ranges;
+static bool s_initialized = false;
+
+unsigned nr_nodes() { return s_nr_nodes; }
+bool available() { return s_available; }
+const std::vector<mem_range>& memory_ranges() { return s_mem_ranges; }
+
+unsigned node_of_cpu(unsigned cpu_id)
+{
+    auto it = s_cpu_to_node.find(cpu_id);
+    return it == s_cpu_to_node.end() ? 0 : it->second;
+}
+
+int node_of_phys(uint64_t phys)
+{
+    for (auto& r : s_mem_ranges) {
+        if (phys >= r.base && phys < r.base + r.length) {
+            return (int)r.node;
+        }
+    }
+    return -1;
+}
+
+unsigned distance(unsigned from, unsigned to)
+{
+    if (from == to) {
+        return 10;   // ACPI convention: 10 == local.
+    }
+    if (!s_distances.empty() && from < s_nr_nodes && to < s_nr_nodes) {
+        return s_distances[from * s_nr_nodes + to];
+    }
+    return 20;       // Default remote distance when no SLIT is present.
+}
+
+#if CONF_drivers_acpi
+using boost::intrusive::get_parent_from_member;
+
+// Record the (apic id -> node) mapping and track the highest node seen.
+static void record_cpu_affinity(uint32_t apic_id, unsigned node, unsigned& max_node)
+{
+    s_apic_to_node[apic_id] = node;
+    max_node = std::max(max_node, node);
+}
+
+static void parse_srat()
+{
+    char sig[] = ACPI_SIG_SRAT;
+    ACPI_TABLE_HEADER* header;
+    if (AcpiGetTable(sig, 0, &header) != AE_OK) {
+        return;   // No SRAT: leave the single-node fallback in place.
+    }
+    auto srat = get_parent_from_member(header, &ACPI_TABLE_SRAT::Header);
+    void* sub = srat + 1;
+    void* end = static_cast<void*>(srat) + srat->Header.Length;
+    unsigned max_node = 0;
+
+    while (sub < end) {
+        auto s = static_cast<ACPI_SUBTABLE_HEADER*>(sub);
+        if (s->Length == 0) {
+            break;   // Guard against a malformed zero-length subtable.
+        }
+        switch (s->Type) {
+        case ACPI_SRAT_TYPE_CPU_AFFINITY: {
+            auto a = get_parent_from_member(s, &ACPI_SRAT_CPU_AFFINITY::Header);
+            if (a->Flags & ACPI_SRAT_CPU_ENABLED) {
+                unsigned node = a->ProximityDomainLo |
+                    (a->ProximityDomainHi[0] << 8) |
+                    (a->ProximityDomainHi[1] << 16) |
+                    (a->ProximityDomainHi[2] << 24);
+                record_cpu_affinity(a->ApicId, node, max_node);
+            }
+            break;
+        }
+        case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: {
+            auto a = get_parent_from_member(s, &ACPI_SRAT_X2APIC_CPU_AFFINITY::Header);
+            if (a->Flags & ACPI_SRAT_CPU_ENABLED) {
+                record_cpu_affinity(a->ApicId, a->ProximityDomain, max_node);
+            }
+            break;
+        }
+        case ACPI_SRAT_TYPE_MEMORY_AFFINITY: {
+            auto m = get_parent_from_member(s, &ACPI_SRAT_MEM_AFFINITY::Header);
+            if (m->Flags & ACPI_SRAT_MEM_ENABLED) {
+                s_mem_ranges.push_back(mem_range{
+                    m->BaseAddress, m->Length, m->ProximityDomain,
+                    (m->Flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) != 0});
+                max_node = std::max(max_node, (unsigned)m->ProximityDomain);
+            }
+            break;
+        }
+        default:
+            break;
+        }
+        sub = static_cast<void*>(sub) + s->Length;
+    }
+
+    if (!s_apic_to_node.empty() || !s_mem_ranges.empty()) {
+        s_available = true;
+        s_nr_nodes = max_node + 1;
+    }
+}
+
+static void parse_slit()
+{
+    char sig[] = ACPI_SIG_SLIT;
+    ACPI_TABLE_HEADER* header;
+    if (AcpiGetTable(sig, 0, &header) != AE_OK) {
+        return;
+    }
+    auto slit = get_parent_from_member(header, &ACPI_TABLE_SLIT::Header);
+    uint64_t n = slit->LocalityCount;
+    // Only trust SLIT if it agrees with the node count we saw in SRAT.
+    if (n == 0 || n != s_nr_nodes) {
+        return;
+    }
+    s_distances.assign(slit->Entry, slit->Entry + n * n);
+}
+
+// Resolve the (apic id -> node) map into a (sched cpu id -> node) map.
+static void resolve_cpus()
+{
+    for (auto* c : sched::cpus) {
+        auto it = s_apic_to_node.find(c->arch.apic_id);
+        if (it != s_apic_to_node.end()) {
+            s_cpu_to_node[c->id] = it->second;
+        }
+    }
+}
+#endif
+
+void init()
+{
+    if (s_initialized) {
+        return;
+    }
+    s_initialized = true;
+
+#if CONF_drivers_acpi
+    parse_srat();
+    if (s_available) {
+        parse_slit();
+        resolve_cpus();
+    }
+#endif
+
+    if (s_available) {
+        debugf("NUMA: %u node(s), %zu CPU(s) mapped, %zu memory range(s)%s\n",
+               s_nr_nodes, s_cpu_to_node.size(), s_mem_ranges.size(),
+               s_distances.empty() ? ", no SLIT" : "");
+    } else {
+        debugf("NUMA: no SRAT, assuming a single flat node\n");
+    }
+}
+
+}

include/osv/numa.hh

@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2026 Waldemar Kozaczuk
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#ifndef OSV_NUMA_HH
+#define OSV_NUMA_HH
+
+#include <cstdint>
+#include <vector>
+#include <cstddef>
+
+// NUMA topology discovery.
+//
+// This module parses the ACPI SRAT (System Resource Affinity Table) and SLIT
+// (System Locality Distance Information Table) to learn the machine's NUMA
+// layout: which node each CPU belongs to, which physical memory ranges belong
+// to each node, and the relative distances between nodes.
+//
+// This is discovery only: it does not change how memory is allocated or how
+// threads are scheduled.  It exposes the topology so later work (a node-aware
+// allocator, scheduler affinity, mbind/get_mempolicy) can use it.  On a machine
+// with no SRAT (the common single-node virtual machine), the whole system is
+// reported as one node (node 0) containing every CPU.
+
+namespace numa {
+
+// A contiguous physical memory range assigned to a NUMA node.
+struct mem_range {
+    uint64_t base;
+    uint64_t length;
+    unsigned node;
+    bool     hotpluggable;
+};
+
+// Discover the topology by parsing SRAT/SLIT.  Safe to call once, after ACPI is
+// initialized and the CPUs have been enumerated (so APIC ids are known).  If no
+// SRAT is present, initializes a single flat node.  Idempotent.
+void init();
+
+// Number of NUMA nodes (>= 1).
+unsigned nr_nodes();
+
+// True if the topology came from a real SRAT (as opposed to the synthesized
+// single-node fallback).
+bool available();
+
+// The node a CPU (by sched cpu id) belongs to, or 0 if unknown.
+unsigned node_of_cpu(unsigned cpu_id);
+
+// The node that owns a physical address, or -1 if it falls in no known range.
+// Only meaningful when available() is true.
+int node_of_phys(uint64_t phys);
+
+// The SLIT distance from node `from` to node `to`.  Linux/ACPI convention:
+// 10 == local (same node), higher == farther.  Returns 10 for the diagonal and
+// a default (10 local / 20 remote) when no SLIT is present.
+unsigned distance(unsigned from, unsigned to);
+
+// The memory ranges discovered from SRAT (empty if none).
+const std::vector<mem_range>& memory_ranges();
+
+}
+
+#endif

linux.cc

@@ -10,6 +10,8 @@
 
 #include <osv/debug.hh>
 #include <osv/sched.hh>
+#include <osv/numa.hh>
+#include <osv/mmu.hh>
 #include <osv/mutex.h>
 #include <osv/waitqueue.hh>
 #include <osv/stubbing.hh>
@@ -176,23 +178,44 @@ int futex(int *uaddr, int op, int val, const struct timespec *timeout,
 static long get_mempolicy(int *policy, unsigned long *nmask,
         unsigned long maxnode, void *addr, int flags)
 {
-    // As OSv has no support for NUMA nodes, we do here the minimum possible,
-    // which is basically to return the same policy (MPOL_DEFAULT) and list
-    // of nodes (just node 0) no matter if the caller asked for the default
-    // policy, the allowed policy, or the policy for a specific address.
+    // Report the topology discovered from ACPI SRAT/SLIT (numa::).  When NUMA is
+    // not available (no SRAT, the common single-node VM) this degrades to the
+    // old single-node-0 behavior.  We still apply MPOL_DEFAULT everywhere: the
+    // allocator is not yet node-aware, so we cannot honor a real policy, but we
+    // can at least answer topology queries truthfully.
     if ((flags & MPOL_F_NODE)) {
-        *policy = 0; // in this case, store a node id, not a policy
+        // Store a node id in *policy, not a policy value.
+        if (!policy) {
+            errno = EINVAL;
+            return -1;
+        }
+        if (flags & MPOL_F_ADDR) {
+            // The node backing the given address, if we can resolve it.
+            int node = -1;
+            if (numa::available() && addr) {
+                auto phys = mmu::virt_to_phys_pt(addr);
+                node = numa::node_of_phys(phys);
+            }
+            *policy = node < 0 ? 0 : node;
+        } else {
+            // The node the calling CPU runs on.
+            *policy = numa::node_of_cpu(sched::cpu::current()->id);
+        }
         return 0;
     }
     if (policy) {
         *policy = MPOL_DEFAULT;
     }
     if (nmask) {
-        if (maxnode < 1) {
+        if (maxnode < numa::nr_nodes()) {
             errno = EINVAL;
             return -1;
         }
-        nmask[0] |= 1;
+        // Set a bit for every node that exists.
+        for (unsigned n = 0; n < numa::nr_nodes(); n++) {
+            nmask[n / (8 * sizeof(unsigned long))] |=
+                1UL << (n % (8 * sizeof(unsigned long)));
+        }
     }
     return 0;
 }
@@ -202,9 +225,11 @@ static long get_mempolicy(int *policy, unsigned long *nmask,
 static long set_mempolicy(int policy, unsigned long *nmask,
         unsigned long maxnode)
 {
-    // OSv has very minimal support for NUMA - merely exposes
-    // all cpus as a single node0 and cannot really apply any meaningful policy
-    // Therefore we implement this as noop, ignore all arguments and return success
+    // The topology is now discovered (numa::), but the physical allocator is
+    // not yet node-aware, so we cannot actually enforce a placement policy.
+    // Accept the call as a no-op (ignoring the requested policy) rather than
+    // failing, so NUMA-aware programs run unmodified.  Enforcement will come
+    // when the allocator learns about nodes.
     return 0;
 }
 #endif
@@ -441,12 +466,13 @@ static long sys_getcwd(char *buf, unsigned long size)
 #define __NR_sys_getcpu __NR_getcpu
 static long sys_getcpu(unsigned int *cpu, unsigned int *node, void *tcache)
 {
+    auto *c = sched::cpu::current();
     if (cpu) {
-        *cpu = sched::cpu::current()->id;
+        *cpu = c->id;
     }
 
     if (node) {
-       *node = 0;
+       *node = numa::node_of_cpu(c->id);
     }
 
     return 0;

loader.cc

@@ -26,6 +26,7 @@
 #endif /* __x86_64__ */
 
 #include <osv/sched.hh>
+#include <osv/numa.hh>
 #include <osv/barrier.hh>
 #include "arch.hh"
 #include "arch-setup.hh"
@@ -824,6 +825,7 @@ void main_cont(int loader_argc, char** loader_argv)
 #ifdef __x86_64__
 #if CONF_drivers_acpi
     acpi::init();
+    numa::init();
 #endif
 #endif /* __x86_64__ */
 

modules/tests/Makefile

@@ -120,6 +120,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-bsd-evh.so \
 	tst-fpu.so tst-preempt.so tst-tracepoint.so tst-hub.so \
 	misc-console.so misc-leak.so misc-readbench.so misc-mmap-anon-perf.so \
 	tst-mmap-file.so misc-mmap-big-file.so tst-mmap.so tst-huge.so \
+	tst-numa.so tst-numa-mempolicy.so \
 	tst-elf-permissions.so misc-mutex.so misc-sockets.so tst-condvar.so \
 	tst-queue-mpsc.so tst-af-local.so tst-pipe.so tst-yield.so \
 	misc-ctxsw.so tst-read.so tst-symlink.so tst-openat.so \