|
| 1 | +// Copyright (c) 2023-2026 Logan McDougall |
| 2 | +// |
| 3 | +// Distributed under the Boost Software License, Version 1.0. (See accompanying |
| 4 | +// file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt) |
| 5 | + |
| 6 | +#pragma once |
| 7 | + |
| 8 | +// CPU timestamp-counter facilities (TMC_CPU_TIMESTAMP / TMC_CPU_FREQ) used by |
| 9 | +// tmc::channel for load-based clustering. Split out of compat.hpp because |
| 10 | +// channel.hpp is the only consumer. The architecture #if/#elif chain below |
| 11 | +// mirrors the one in compat.hpp; when adding support for a new architecture, |
| 12 | +// update both. |
| 13 | + |
| 14 | +#include <cstddef> |
| 15 | +#include <cstdint> // IWYU pragma: keep |
| 16 | + |
| 17 | +#if defined(__x86_64__) || defined(_M_AMD64) || defined(i386) || defined(__i386__) || \ |
| 18 | + defined(__i386) || defined(_M_IX86) |
| 19 | +#ifdef _MSC_VER |
| 20 | +#include <intrin.h> |
| 21 | +#else |
| 22 | +#include <immintrin.h> |
| 23 | +#endif |
| 24 | +static inline size_t TMC_CPU_TIMESTAMP() noexcept { |
| 25 | + return static_cast<size_t>(__rdtsc()); |
| 26 | +} |
| 27 | +// Assume a 3.5GHz CPU if we can't get the value (on x86). |
| 28 | +// Yes, this is hacky. Getting the real RDTSC freq requires |
| 29 | +// waiting for another time source (system timer) and then dividing by |
| 30 | +// that duration. This takes real time and would have to be done on |
| 31 | +// startup. Using a 3.5GHz default means that slower processors will appear to |
| 32 | +// be running faster, and vice versa. For the current usage of this (the |
| 33 | +// clustering threshold in tmc::channel) this seems like reasonable behavior |
| 34 | +// anyway. |
| 35 | +static inline const size_t TMC_CPU_FREQ = 3500000000; |
| 36 | +#elif defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) |
| 37 | +#ifdef _MSC_VER |
| 38 | +#include <intrin.h> |
| 39 | +#endif |
| 40 | +#if defined(_MSC_VER) && defined(_M_ARM64) |
| 41 | +#define TMC_ARM64_SYSREG(op0, op1, crn, crm, op2) \ |
| 42 | + (((op0 & 1) << 14) | ((op1 & 7) << 11) | ((crn & 15) << 7) | ((crm & 15) << 3) | \ |
| 43 | + ((op2 & 7) << 0)) |
| 44 | +static inline size_t TMC_CPU_TIMESTAMP() noexcept { |
| 45 | + return static_cast<size_t>(_ReadStatusReg(TMC_ARM64_SYSREG(3, 3, 14, 0, 2))); |
| 46 | +} |
| 47 | +static inline size_t TMC_ARM_CPU_FREQ() noexcept { |
| 48 | + return static_cast<size_t>(_ReadStatusReg(TMC_ARM64_SYSREG(3, 3, 14, 0, 0))); |
| 49 | +} |
| 50 | +#undef TMC_ARM64_SYSREG |
| 51 | +#elif defined(_MSC_VER) && defined(_M_ARM) |
| 52 | +static inline size_t TMC_CPU_TIMESTAMP() noexcept { |
| 53 | + return static_cast<size_t>(_MoveFromCoprocessor64(15, 1, 14)); |
| 54 | +} |
| 55 | +static inline size_t TMC_ARM_CPU_FREQ() noexcept { |
| 56 | + return static_cast<size_t>(_MoveFromCoprocessor(15, 0, 14, 0, 0)); |
| 57 | +} |
| 58 | +#elif defined(__aarch64__) || defined(_M_ARM64) |
| 59 | +// AArch64: the generic timer is read through dedicated system registers. |
| 60 | +// Read the ARM "Virtual Counter" register. |
| 61 | +// This ticks at a frequency independent of the processor frequency. |
| 62 | +// https://developer.arm.com/documentation/ddi0406/cb/System-Level-Architecture/The-Generic-Timer/About-the-Generic-Timer/The-virtual-counter?lang=en |
| 63 | +static inline size_t TMC_CPU_TIMESTAMP() noexcept { |
| 64 | + size_t count; |
| 65 | + asm volatile("mrs %0, cntvct_el0; " : "=r"(count)::"memory"); |
| 66 | + return count; |
| 67 | +} |
| 68 | +// Read the ARM "Virtual Counter" frequency. |
| 69 | +static inline size_t TMC_ARM_CPU_FREQ() noexcept { |
| 70 | + size_t freq; |
| 71 | + asm volatile("mrs %0, cntfrq_el0; isb; " : "=r"(freq)::"memory"); |
| 72 | + return freq; |
| 73 | +} |
| 74 | +#elif defined(__ARM_ARCH) && __ARM_ARCH >= 7 && defined(__ARM_ARCH_PROFILE) && \ |
| 75 | + (__ARM_ARCH_PROFILE == 'A' || __ARM_ARCH_PROFILE == 'R') |
| 76 | +#if defined(__linux__) |
| 77 | +#include <asm/hwcap.h> |
| 78 | +#include <sys/auxv.h> |
| 79 | +#endif |
| 80 | +// AArch32 (e.g. armhf): the AArch64 `cntvct_el0` / `cntfrq_el0` system |
| 81 | +// registers don't exist here. The ARMv7-A Generic Timer exposes the same |
| 82 | +// counters through CP15 coprocessor accesses instead. |
| 83 | +// https://developer.arm.com/documentation/ddi0406/cb/System-Level-Architecture/The-Generic-Timer/Register-descriptions?lang=en |
| 84 | +// The Generic Timer is optional on ARMv7-A, and user-mode access is controlled |
| 85 | +// by the OS. On Linux, HWCAP_EVTSTRM indicates that the kernel configured the |
| 86 | +// architected timer event stream; if that is absent, fall back to inert timing |
| 87 | +// rather than trapping on an undefined/privileged CP15 access. |
| 88 | +static inline bool TMC_ARM_HAS_GENERIC_TIMER_IMPL() noexcept { |
| 89 | +#if defined(__linux__) && defined(HWCAP_EVTSTRM) |
| 90 | + return (getauxval(AT_HWCAP) & HWCAP_EVTSTRM) != 0; |
| 91 | +#else |
| 92 | + return false; |
| 93 | +#endif |
| 94 | +} |
| 95 | +static inline const bool TMC_ARM_HAS_GENERIC_TIMER = TMC_ARM_HAS_GENERIC_TIMER_IMPL(); |
| 96 | +// Read the 64-bit "Virtual Counter" (CNTVCT) via MRRC into a register pair. |
| 97 | +// size_t is 32-bit here, so we return only the low word - matching the 32-bit |
| 98 | +// x86 path, which likewise truncates __rdtsc(). The counter ticks at a |
| 99 | +// frequency independent of the processor frequency. |
| 100 | +static inline size_t TMC_CPU_TIMESTAMP() noexcept { |
| 101 | + if (!TMC_ARM_HAS_GENERIC_TIMER) { |
| 102 | + return 0; |
| 103 | + } |
| 104 | + uint32_t lo, hi; |
| 105 | + asm volatile("mrrc p15, 1, %0, %1, c14" : "=r"(lo), "=r"(hi)::"memory"); |
| 106 | + (void)hi; |
| 107 | + return lo; |
| 108 | +} |
| 109 | +// Read the "Virtual Counter" frequency (CNTFRQ, a 32-bit register) via MRC. |
| 110 | +static inline size_t TMC_ARM_CPU_FREQ() noexcept { |
| 111 | + if (!TMC_ARM_HAS_GENERIC_TIMER) { |
| 112 | + return 1000000000; |
| 113 | + } |
| 114 | + uint32_t freq; |
| 115 | + asm volatile("mrc p15, 0, %0, c14, c0, 0" : "=r"(freq)::"memory"); |
| 116 | + return freq; |
| 117 | +} |
| 118 | +#else |
| 119 | +static inline size_t TMC_CPU_TIMESTAMP() noexcept { return 0; } |
| 120 | +static inline size_t TMC_ARM_CPU_FREQ() noexcept { return 1000000000; } |
| 121 | +#endif |
| 122 | +static inline const size_t TMC_CPU_FREQ = TMC_ARM_CPU_FREQ(); |
| 123 | +#elif defined(__loongarch__) && defined(__LP64__) |
| 124 | +// Read the LoongArch stable counter |
| 125 | +static inline size_t TMC_CPU_TIMESTAMP() noexcept { |
| 126 | + size_t count; |
| 127 | + asm volatile("rdtime.d %0, $zero" : "=r"(count)); |
| 128 | + return count; |
| 129 | +} |
| 130 | +// Calculate the LoongArch stable counter frequency |
| 131 | +static inline size_t TMC_LOONGARCH_CPU_FREQ() noexcept { |
| 132 | + size_t cc_freq; |
| 133 | + asm("cpucfg %0, %1" : "=r"(cc_freq) : "r"(0x4)); |
| 134 | + |
| 135 | + size_t cc_mul_div; |
| 136 | + asm("cpucfg %0, %1" : "=r"(cc_mul_div) : "r"(0x5)); |
| 137 | + |
| 138 | + size_t cc_mul = cc_mul_div & 0xffff; |
| 139 | + size_t cc_div = (cc_mul_div >> 16) & 0xffff; |
| 140 | + |
| 141 | + return cc_freq * cc_mul / cc_div; |
| 142 | +} |
| 143 | + |
| 144 | +static inline const size_t TMC_CPU_FREQ = TMC_LOONGARCH_CPU_FREQ(); |
| 145 | +#elif defined(__riscv) |
| 146 | +#if defined(__linux__) |
| 147 | +#include <cstdio> |
| 148 | +#endif |
| 149 | +static inline size_t TMC_RISCV_READ_TIMEBASE_FREQ() noexcept { |
| 150 | +#if defined(__linux__) |
| 151 | + const char* paths[] = { |
| 152 | + "/proc/device-tree/cpus/timebase-frequency", |
| 153 | + "/sys/firmware/devicetree/base/cpus/timebase-frequency", |
| 154 | + }; |
| 155 | + for (const char* path : paths) { |
| 156 | + auto* file = std::fopen(path, "rb"); |
| 157 | + if (file == nullptr) { |
| 158 | + continue; |
| 159 | + } |
| 160 | + unsigned char bytes[8] = {}; |
| 161 | + const auto size = std::fread(bytes, 1, sizeof(bytes), file); |
| 162 | + std::fclose(file); |
| 163 | + if (size == 4 || size == 8) { |
| 164 | + size_t result = 0; |
| 165 | + for (size_t i = 0; i != size; ++i) { |
| 166 | + result = (result << 8) | bytes[i]; |
| 167 | + } |
| 168 | + if (result != 0) { |
| 169 | + return result; |
| 170 | + } |
| 171 | + } |
| 172 | + } |
| 173 | +#endif |
| 174 | + // This is only used for heuristics if the real RISC-V timebase is unavailable. |
| 175 | + // Use the 2GHz clock frequency of SG2042-class server chips as a default. |
| 176 | + return 2000000000; |
| 177 | +} |
| 178 | +static inline size_t TMC_CPU_TIMESTAMP() noexcept { |
| 179 | + size_t count; |
| 180 | + asm volatile("rdtime %0" : "=r"(count)); |
| 181 | + return count; |
| 182 | +} |
| 183 | +static inline const size_t TMC_CPU_FREQ = TMC_RISCV_READ_TIMEBASE_FREQ(); |
| 184 | +#else |
| 185 | +// Fallback for unknown architectures |
| 186 | +static inline size_t TMC_CPU_TIMESTAMP() noexcept { return 0; } |
| 187 | +static inline const size_t TMC_CPU_FREQ = 1000000000; |
| 188 | +#endif |
0 commit comments