nvidia: add voltage via nvapi

Author: 17314642

server/metrics/gpu/gpu.cpp

@@ -90,6 +90,10 @@ GPUS::GPUS() {
 
                     continue;
                 }
+
+                if (!ptr->nvapi_available) {
+                    SPDLOG_WARN("NvAPI is not loaded. Voltage might not be available");
+                }
             }
         } else if (driver == "panfrost") {
             gpu = std::make_shared<Panfrost>(drm_node, pci_dev, vendor_id, device_id);

server/metrics/gpu/meson.build

@@ -13,7 +13,8 @@ gpu_lib = static_library(
           'msm/dpu.cpp',
           'msm/kgsl.cpp',
           'nvidia/nvidia.cpp',
-          'nvidia/nvml_loader.cpp'),
+          'nvidia/nvml_loader.cpp',
+          'nvidia/nvapi_loader.cpp'),
     include_directories: '../',
     dependencies: [libdrm_dep,
                    libcap_dep,

server/metrics/gpu/nvidia/nvapi_loader.cpp

@@ -0,0 +1,78 @@
+#include "nvapi_loader.hpp"
+#include <dlfcn.h>
+#include <spdlog/spdlog.h>
+
+static std::shared_ptr<libnvapi_loader> libnvapi_;
+
+std::shared_ptr<libnvapi_loader> get_libnvapi_loader()
+{
+    if (!libnvapi_)
+        libnvapi_ = std::make_shared<libnvapi_loader>();
+
+    return libnvapi_;
+}
+
+libnvapi_loader::libnvapi_loader() {
+    load();
+}
+
+libnvapi_loader::~libnvapi_loader() {
+    unload();
+}
+
+#define LOAD_NVAPI_FUNCTION(name, id) \
+    do {                    \
+        name = reinterpret_cast<decltype(this->name)>(nvapi_QueryInterface(id)); \
+        if (!name) { \
+            SPDLOG_ERROR("Failed to find address of {}", #name); \
+            unload(); \
+            return false; \
+        } \
+    } while(0)
+
+bool libnvapi_loader::load() {
+    if (loaded_)
+        return true;
+
+    library_ = dlopen(library_name.c_str(), RTLD_LAZY | RTLD_NODELETE);
+
+    if (!library_) {
+        SPDLOG_ERROR("Failed to open {}: {}", library_name, dlerror());
+        return false;
+    }
+
+    nvapi_QueryInterface =
+        reinterpret_cast<decltype(nvapi_QueryInterface)>(dlsym(library_, "nvapi_QueryInterface"));
+
+    if (!nvapi_QueryInterface) {
+        SPDLOG_ERROR("Failed to find address of nvapi_QueryInterface");
+        return false;
+    }
+
+    LOAD_NVAPI_FUNCTION(nvapi_Initialize, NVAPI_INIT_ID);
+    LOAD_NVAPI_FUNCTION(nvapi_GetErrorMessage, NVAPI_GET_ERROR_MESSAGE);
+    LOAD_NVAPI_FUNCTION(nvapi_EnumPhysicalGPUs, NVAPI_ENUM_PHYSICAL_GPUS);
+    LOAD_NVAPI_FUNCTION(nvapi_GPU_GetBusId, NVAPI_GET_BUS_ID);
+    LOAD_NVAPI_FUNCTION(nvapi_GetVoltage, NVAPI_GET_CURRENT_VOLTAGE);
+
+    loaded_ = true;
+    return true;
+}
+
+#undef LOAD_NVAPI_FUNCTION
+
+void libnvapi_loader::unload() {
+    if (library_) {
+        dlclose(library_);
+        library_ = NULL;
+    }
+
+    loaded_ = false;
+
+    nvapi_QueryInterface = nullptr;
+    nvapi_Initialize = nullptr;
+    nvapi_GetErrorMessage = nullptr;
+    nvapi_EnumPhysicalGPUs = nullptr;
+    nvapi_GPU_GetBusId = nullptr;
+    nvapi_GetVoltage = nullptr;
+}

server/metrics/gpu/nvidia/nvapi_loader.hpp

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <string>
+#include <memory>
+#include <atomic>
+
+#define NVAPI_MAX_PHYSICAL_GPUS 64
+#define NVAPI_INIT_ID 0x0150e828
+#define NVAPI_GET_ERROR_MESSAGE 0x6c2d048c
+#define NVAPI_ENUM_PHYSICAL_GPUS 0xe5ac921f
+#define NVAPI_GET_BUS_ID 0x1be0b8e5
+#define NVAPI_GET_CURRENT_VOLTAGE 0x465f9bcf
+
+class libnvapi_loader {
+public:
+    libnvapi_loader();
+    ~libnvapi_loader();
+
+    struct NvApiVoltage {
+        unsigned int version = sizeof(NvApiVoltage) | (1 << 16);
+        unsigned int flags;
+        unsigned int padding_1[8];
+        unsigned int value_microvolts;
+        unsigned int padding_2[8];
+    };
+
+    bool load();
+    bool is_loaded() { return loaded_; }
+
+    void* (*nvapi_QueryInterface)(unsigned int);
+    int (*nvapi_Initialize)();
+    int (*nvapi_GetErrorMessage)(int, char(*)[64]);
+    int (*nvapi_EnumPhysicalGPUs)(void*, unsigned int*);
+    int (*nvapi_GPU_GetBusId)(long, unsigned int*);
+    int (*nvapi_GetVoltage)(long, NvApiVoltage*);
+
+private:
+    void unload();
+    void* library_ = nullptr;
+
+    const std::string library_name = "libnvidia-api.so.1";
+    std::atomic<bool> loaded_ = false;
+
+    // Disallow copy constructor and assignment operator.
+    libnvapi_loader(const libnvapi_loader&);
+    void operator=(const libnvapi_loader&);
+};
+
+std::shared_ptr<libnvapi_loader> get_libnvapi_loader();

server/metrics/gpu/nvidia/nvidia.cpp

@@ -7,6 +7,7 @@ Nvidia::Nvidia(
     GPU(drm_node, pci_dev, vendor_id, device_id, "gpu-nvidia"
 ) {
     nvml_available = init_nvml(pci_dev);
+    nvapi_available = init_nvapi(pci_dev);
 }
 
 bool Nvidia::init_nvml(const std::string& pci_dev) {
@@ -35,6 +36,67 @@ bool Nvidia::init_nvml(const std::string& pci_dev) {
     return true;
 }
 
+bool Nvidia::init_nvapi(const std::string& pci_dev) {
+    nvapi = get_libnvapi_loader();
+
+    unsigned int pciBusId = 0;
+
+    {
+        unsigned int domain, bus, slot, func;
+
+        if (sscanf(pci_dev.c_str(), "%x:%02x:%02x.%x", &domain, &bus, &slot, &func) != 4) {
+            SPDLOG_ERROR("nvapi: Failed to parse PCI device ID: '{}'", pci_dev);
+            return false;
+        }
+
+        pciBusId = bus;
+    }
+
+    if (!nvapi)
+        return false;
+
+    if (!nvapi->is_loaded())
+        return false;
+
+    int result = nvapi->nvapi_Initialize();
+
+    if (result != 0) {
+        char msg[64] = {};
+        nvapi->nvapi_GetErrorMessage(result, &msg);
+        SPDLOG_ERROR("nvapi_Initialize() failed: {}", msg);
+        return false;
+    }
+
+    long nvGPUHandle[NVAPI_MAX_PHYSICAL_GPUS] = {};
+    unsigned int numOfGPUs = 0;
+
+    result = nvapi->nvapi_EnumPhysicalGPUs(&nvGPUHandle, &numOfGPUs);
+
+    if (result != 0) {
+        char msg[64] = {};
+        nvapi->nvapi_GetErrorMessage(result, &msg);
+        SPDLOG_ERROR("nvapi_EnumPhysicalGPUs() failed: {}", msg);
+        return false;
+    }
+
+    for (unsigned int i = 0; i < numOfGPUs; i++) {
+        unsigned int busId = 0;
+        result = nvapi->nvapi_GPU_GetBusId(nvGPUHandle[i], &busId);
+
+        if (result == 0 && busId == pciBusId) {
+            nvapi_device = nvGPUHandle[i];
+            break;
+        }
+    }
+
+    if (nvapi_device == 0) {
+        SPDLOG_ERROR("nvapi: Failed to find gpu with {}", pci_dev);
+        return false;
+    }
+
+    return true;
+}
+
 const std::vector<nvmlProcessInfo_v1_t> Nvidia::get_processes() {
     unsigned int info_count = 0;
 
@@ -121,15 +183,27 @@ int Nvidia::get_core_clock() {
 
 int Nvidia::get_voltage() {
     uint32_t voltage = 0;
+    bool try_nvapi = false;
 
-    if (nvml->nvmlInternalGetVoltage == nullptr) {
-        return 0;
+    if (nvml->nvmlInternalGetVoltage != nullptr) {
+        nvmlReturn_t ret = nvml->nvmlInternalGetVoltage(device, &voltage);
+
+        if (ret != NVML_SUCCESS) {
+            try_nvapi = true;
+        }
+    } else {
+        try_nvapi = true;
     }
 
-    nvmlReturn_t ret = nvml->nvmlInternalGetVoltage(device, &voltage);
+    if (try_nvapi) {
+        libnvapi_loader::NvApiVoltage voltage_info = {};
 
-    if (ret != NVML_SUCCESS)
-        return 0;
+        int nv_ret = nvapi->nvapi_GetVoltage(nvapi_device, &voltage_info);
+        
+        if (nv_ret == 0) {
+            voltage = voltage_info.value_microvolts;
+        }
+    }
 
     return voltage / 1000.f;
 }

server/metrics/gpu/nvidia/nvidia.hpp

@@ -4,13 +4,18 @@
 #include <utility>
 #include "gpu.hpp"
 #include "nvml_loader.hpp"
+#include "nvapi_loader.hpp"
 
 class Nvidia : public GPU {
 private:
     std::shared_ptr<libnvml_loader> nvml;
     nvmlDevice_t device = nullptr;
     bool init_nvml(const std::string& pci_dev);
 
+    std::shared_ptr<libnvapi_loader> nvapi;
+    uint32_t nvapi_device = 0;
+    bool init_nvapi(const std::string& pci_dev);
+
     const std::vector<nvmlProcessInfo_v1_t> get_processes();
 
 public:
@@ -20,6 +25,7 @@ class Nvidia : public GPU {
     );
 
     bool nvml_available = false;
+    bool nvapi_available = false;
 
     // System-related functions
     int     get_load()          override;