Skip to content

Commit d2d3a5f

Browse files
committed
Add configurable hugepage optimization to system allocator
- Add madvise_hugepage_mmap_enabled parameter (default false) - Remove size gating per @ckennelly's feedback (TCMalloc operates in hugepage chunks) - Advisory-only madvise calls with error restoration - Runtime configurable via TCMalloc_Internal_SetMadviseHugepageMmapEnabled() - No behavior change unless explicitly enabled The optimization opts into transparent hugepages when the system is configured for transparent_hugepage=madvise. This can improve memory performance by reducing TLB pressure for large allocations. Follows TCMalloc parameter patterns with atomic storage and weak function accessors for runtime configuration.
1 parent d7bcc13 commit d2d3a5f

4 files changed

Lines changed: 26 additions & 0 deletions

File tree

tcmalloc/internal/parameter_accessors.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetUseUserspaceCollapseHeuristics(
109109
ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetHeapPartitioning();
110110
ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetBackSmallAllocations();
111111
ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetBackSmallAllocations(bool v);
112+
ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetMadviseHugepageMmapEnabled();
113+
ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetMadviseHugepageMmapEnabled(bool v);
112114
ABSL_ATTRIBUTE_WEAK int32_t TCMalloc_Internal_GetBackSizeThresholdBytes();
113115
ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v);
114116
ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetEnableUnfilteredCollapse();

tcmalloc/internal/system_allocator.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,13 @@ SystemAllocator<Topology, NormalPartitions>::MmapRegion::Alloc(
377377
// This is only advisory, so ignore the error.
378378
ErrnoRestorer errno_restorer;
379379
(void)madvise(result_ptr, actual_size, MADV_NOHUGEPAGE);
380+
} else if (Parameters::madvise_hugepage_mmap_enabled()) {
381+
// Opt-in to transparent hugepages when system is
382+
// configured for transparent_hugepage=madvise. This can improve memory
383+
// performance by reducing TLB pressure.
384+
// This is only advisory, so ignore the error.
385+
ErrnoRestorer errno_restorer;
386+
(void)madvise(result_ptr, actual_size, MADV_HUGEPAGE);
380387
}
381388
free_size_ -= actual_size;
382389
return {result_ptr, actual_size};

tcmalloc/parameters.cc

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,8 @@ ABSL_CONST_INIT std::atomic<int32_t> Parameters::back_size_threshold_bytes_(
244244
kPageSize);
245245
ABSL_CONST_INIT std::atomic<bool> Parameters::enable_unfiltered_collapse_(
246246
false);
247+
ABSL_CONST_INIT std::atomic<bool> Parameters::madvise_hugepage_mmap_enabled_(
248+
false);
247249

248250
static std::atomic<bool>& back_small_allocations_enabled() {
249251
ABSL_CONST_INIT static absl::once_flag flag;
@@ -335,6 +337,11 @@ int32_t Parameters::max_per_cpu_cache_size() {
335337
return tc_globals.cpu_cache().CacheLimit();
336338
}
337339

340+
bool TCMalloc_Internal_GetMadviseHugepageMmapEnabled() {
341+
return Parameters::madvise_hugepage_mmap_enabled();
342+
}
343+
344+
338345
int ABSL_ATTRIBUTE_WEAK default_want_disable_dynamic_slabs();
339346

340347
// TODO(b/271475288): remove the default_want_disable_dynamic_slabs opt-out
@@ -642,6 +649,10 @@ void TCMalloc_Internal_SetBackSmallAllocations(bool v) {
642649
v, std::memory_order_relaxed);
643650
}
644651

652+
void TCMalloc_Internal_SetMadviseHugepageMmapEnabled(bool v) {
653+
Parameters::madvise_hugepage_mmap_enabled_.store(v, std::memory_order_relaxed);
654+
}
655+
645656
void TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v) {
646657
Parameters::back_size_threshold_bytes_.store(v, std::memory_order_relaxed);
647658
}

tcmalloc/parameters.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,10 @@ class Parameters {
185185
TCMalloc_Internal_SetPerCpuCachesDynamicSlabShrinkThreshold(value);
186186
}
187187

188+
static bool madvise_hugepage_mmap_enabled() {
189+
return madvise_hugepage_mmap_enabled_.load(std::memory_order_relaxed);
190+
}
191+
188192
static bool heap_partitioning();
189193

190194
static central_freelist_internal::LifetimeTracking span_lifetime_tracking();
@@ -224,6 +228,7 @@ class Parameters {
224228
friend void ::TCMalloc_Internal_SetBackSmallAllocations(bool v);
225229
friend void ::TCMalloc_Internal_SetBackSizeThresholdBytes(int32_t v);
226230
friend void ::TCMalloc_Internal_SetEnableUnfilteredCollapse(bool v);
231+
friend void ::TCMalloc_Internal_SetMadviseHugepageMmapEnabled(bool v);
227232

228233
static std::atomic<int64_t> guarded_sampling_interval_;
229234
static std::atomic<int32_t> max_per_cpu_cache_size_;
@@ -243,6 +248,7 @@ class Parameters {
243248
static std::atomic<bool> use_userspace_collapse_heuristics_;
244249
static std::atomic<int32_t> back_size_threshold_bytes_;
245250
static std::atomic<bool> enable_unfiltered_collapse_;
251+
static std::atomic<bool> madvise_hugepage_mmap_enabled_;
246252
};
247253

248254
} // namespace tcmalloc_internal

0 commit comments

Comments
 (0)