Skip to content

Commit d7bcc13

Browse files
ckennellycopybara-github
ckennelly
authored and
copybara-github
committed
Shift CFL hook state to a single library.
This keeps it out of TCMalloc and avoids a dependency on TCMalloc's build size for any library that uses the hooks. PiperOrigin-RevId: 899598246 Change-Id: I499b5a2c8783304dacddb08e2261d145e17c7328
1 parent ba72dfc commit d7bcc13

13 files changed

Lines changed: 189 additions & 41 deletions

‎tcmalloc/BUILD‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ tcmalloc_deps = [
8383
"@com_google_absl//absl/strings:resize_and_overwrite",
8484
"@com_google_absl//absl/strings:str_format",
8585
"@com_google_absl//absl/numeric:bits",
86+
"//tcmalloc/internal:central_freelist_hooks",
8687
"//tcmalloc/internal:config",
8788
"//tcmalloc/internal:declarations",
8889
"//tcmalloc/internal:linked_list",
@@ -288,6 +289,7 @@ create_tcmalloc_libraries(
288289
"//tcmalloc/internal:allocation_guard",
289290
"//tcmalloc/internal:atomic_stats_counter",
290291
"//tcmalloc/internal:cache_topology",
292+
"//tcmalloc/internal:central_freelist_hooks",
291293
"//tcmalloc/internal:clock",
292294
"//tcmalloc/internal:config",
293295
"//tcmalloc/internal:cpu_utils",
@@ -641,9 +643,14 @@ cc_library(
641643
testonly = 1,
642644
hdrs = ["mock_static_forwarder.h"],
643645
copts = TCMALLOC_DEFAULT_COPTS,
646+
visibility = [
647+
"//tcmalloc:__subpackages__",
648+
],
644649
deps = [
645650
":common_8k_pages",
651+
"//tcmalloc/internal:config",
646652
"//tcmalloc/internal:hook_list",
653+
"@com_google_absl//absl/base:core_headers",
647654
"@com_google_absl//absl/synchronization",
648655
"@com_google_absl//absl/time",
649656
"@com_google_absl//absl/types:span",

‎tcmalloc/CMakeLists.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ tcmalloc_cc_library(
6565
"tcmalloc::common_8k_pages"
6666
"tcmalloc::experiment"
6767
"tcmalloc::internal_allocation_guard"
68+
"tcmalloc::internal_central_freelist_hooks"
6869
"tcmalloc::internal_config"
6970
"tcmalloc::internal_declarations"
7071
"tcmalloc::internal_linked_list"
@@ -111,6 +112,7 @@ tcmalloc_cc_library(
111112
"tcmalloc::common_8k_pages"
112113
"tcmalloc::experiment"
113114
"tcmalloc::internal_allocation_guard"
115+
"tcmalloc::internal_central_freelist_hooks"
114116
"tcmalloc::internal_config"
115117
"tcmalloc::internal_declarations"
116118
"tcmalloc::internal_linked_list"
@@ -157,6 +159,7 @@ tcmalloc_cc_library(
157159
"tcmalloc::common_deprecated_perthread"
158160
"tcmalloc::experiment"
159161
"tcmalloc::internal_allocation_guard"
162+
"tcmalloc::internal_central_freelist_hooks"
160163
"tcmalloc::internal_config"
161164
"tcmalloc::internal_declarations"
162165
"tcmalloc::internal_linked_list"
@@ -305,6 +308,7 @@ tcmalloc_cc_library_variants(
305308
"tcmalloc::internal_allocation_guard"
306309
"tcmalloc::internal_atomic_stats_counter"
307310
"tcmalloc::internal_cache_topology"
311+
"tcmalloc::internal_central_freelist_hooks"
308312
"tcmalloc::internal_clock"
309313
"tcmalloc::internal_config"
310314
"tcmalloc::internal_cpu_utils"
@@ -426,6 +430,7 @@ tcmalloc_cc_library(
426430
"tcmalloc::common_large_pages"
427431
"tcmalloc::experiment"
428432
"tcmalloc::internal_allocation_guard"
433+
"tcmalloc::internal_central_freelist_hooks"
429434
"tcmalloc::internal_config"
430435
"tcmalloc::internal_declarations"
431436
"tcmalloc::internal_linked_list"
@@ -472,6 +477,7 @@ tcmalloc_cc_library(
472477
"tcmalloc::common_256k_pages"
473478
"tcmalloc::experiment"
474479
"tcmalloc::internal_allocation_guard"
480+
"tcmalloc::internal_central_freelist_hooks"
475481
"tcmalloc::internal_config"
476482
"tcmalloc::internal_declarations"
477483
"tcmalloc::internal_linked_list"
@@ -519,6 +525,7 @@ tcmalloc_cc_library(
519525
"tcmalloc::common_256k_pages_numa_aware"
520526
"tcmalloc::experiment"
521527
"tcmalloc::internal_allocation_guard"
528+
"tcmalloc::internal_central_freelist_hooks"
522529
"tcmalloc::internal_config"
523530
"tcmalloc::internal_declarations"
524531
"tcmalloc::internal_linked_list"
@@ -565,6 +572,7 @@ tcmalloc_cc_library(
565572
"tcmalloc::common_small_but_slow"
566573
"tcmalloc::experiment"
567574
"tcmalloc::internal_allocation_guard"
575+
"tcmalloc::internal_central_freelist_hooks"
568576
"tcmalloc::internal_config"
569577
"tcmalloc::internal_declarations"
570578
"tcmalloc::internal_linked_list"
@@ -612,6 +620,7 @@ tcmalloc_cc_library(
612620
"tcmalloc::common_small_but_slow_with_assertions"
613621
"tcmalloc::experiment"
614622
"tcmalloc::internal_allocation_guard"
623+
"tcmalloc::internal_central_freelist_hooks"
615624
"tcmalloc::internal_config"
616625
"tcmalloc::internal_declarations"
617626
"tcmalloc::internal_linked_list"
@@ -658,6 +667,7 @@ tcmalloc_cc_library(
658667
"tcmalloc::common_numa_aware"
659668
"tcmalloc::experiment"
660669
"tcmalloc::internal_allocation_guard"
670+
"tcmalloc::internal_central_freelist_hooks"
661671
"tcmalloc::internal_config"
662672
"tcmalloc::internal_declarations"
663673
"tcmalloc::internal_linked_list"
@@ -705,6 +715,7 @@ tcmalloc_cc_library(
705715
"tcmalloc::common_legacy_locking"
706716
"tcmalloc::experiment"
707717
"tcmalloc::internal_allocation_guard"
718+
"tcmalloc::internal_central_freelist_hooks"
708719
"tcmalloc::internal_config"
709720
"tcmalloc::internal_declarations"
710721
"tcmalloc::internal_linked_list"
@@ -752,6 +763,7 @@ tcmalloc_cc_library(
752763
"tcmalloc::common_latency_injection"
753764
"tcmalloc::experiment"
754765
"tcmalloc::internal_allocation_guard"
766+
"tcmalloc::internal_central_freelist_hooks"
755767
"tcmalloc::internal_config"
756768
"tcmalloc::internal_declarations"
757769
"tcmalloc::internal_delay_injection"
@@ -797,10 +809,12 @@ tcmalloc_cc_library(
797809
DEPS
798810
"GTest::gtest"
799811
"GTest::gmock"
812+
"absl::core_headers"
800813
"absl::span"
801814
"absl::synchronization"
802815
"absl::time"
803816
"tcmalloc::common_8k_pages"
817+
"tcmalloc::internal_config"
804818
"tcmalloc::internal_hook_list"
805819
)
806820

‎tcmalloc/central_freelist.cc‎

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "tcmalloc/common.h"
2828
#include "tcmalloc/error_reporting.h"
2929
#include "tcmalloc/internal/allocation_guard.h"
30+
#include "tcmalloc/internal/central_freelist_hooks.h"
3031
#include "tcmalloc/internal/config.h"
3132
#include "tcmalloc/internal/hook_list.h"
3233
#include "tcmalloc/internal/logging.h"
@@ -47,26 +48,6 @@ namespace tcmalloc {
4748
namespace tcmalloc_internal {
4849
namespace central_freelist_internal {
4950

50-
static void RemoveInitialHooksAndCallInitializers();
51-
52-
static void InitialRemoveRangeHook(size_t size_class, absl::Span<void*> batch) {
53-
ABSL_CONST_INIT static absl::once_flag once;
54-
absl::base_internal::LowLevelCallOnce(&once,
55-
RemoveInitialHooksAndCallInitializers);
56-
StaticForwarder::InvokeRemoveRangeHook(size_class, batch);
57-
}
58-
59-
ABSL_CONST_INIT HookList<InsertRangeHook> StaticForwarder::insert_range_hooks;
60-
ABSL_CONST_INIT HookList<RemoveRangeHook> StaticForwarder::remove_range_hooks{
61-
&InitialRemoveRangeHook};
62-
63-
void RemoveInitialHooksAndCallInitializers() {
64-
TC_CHECK(StaticForwarder::remove_range_hooks.Remove(&InitialRemoveRangeHook));
65-
if (TCMalloc_CentralFreeList_InitAtFirstRemoveRange_Tracing != nullptr) {
66-
TCMalloc_CentralFreeList_InitAtFirstRemoveRange_Tracing();
67-
}
68-
}
69-
7051
static MemoryTag MemoryTagFromSizeClass(size_t size_class) {
7152
if (IsExpandedSizeClass(size_class)) {
7253
return MemoryTag::kCold;
@@ -208,12 +189,12 @@ void StaticForwarder::DeallocateSpans(size_t objects_per_span,
208189

209190
ABSL_ATTRIBUTE_NOINLINE void StaticForwarder::InvokeInsertRangeHookSlow(
210191
size_t size_class, absl::Span<void*> batch) {
211-
insert_range_hooks.Invoke(size_class, batch);
192+
central_freelist_insert_range_hooks.Invoke(size_class, batch);
212193
}
213194

214195
ABSL_ATTRIBUTE_NOINLINE void StaticForwarder::InvokeRemoveRangeHookSlow(
215196
size_t size_class, absl::Span<void*> batch) {
216-
remove_range_hooks.Invoke(size_class, batch);
197+
central_freelist_remove_range_hooks.Invoke(size_class, batch);
217198
}
218199

219200
} // namespace central_freelist_internal

‎tcmalloc/central_freelist.h‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
#include "tcmalloc/common.h"
3737
#include "tcmalloc/hinted_tracker_lists.h"
3838
#include "tcmalloc/internal/atomic_stats_counter.h"
39+
#include "tcmalloc/internal/central_freelist_hooks.h"
3940
#include "tcmalloc/internal/config.h"
4041
#include "tcmalloc/internal/delay_injection.h"
4142
#include "tcmalloc/internal/hook_list.h"
@@ -79,20 +80,17 @@ using RemoveRangeHook = void (*)(size_t size_class, absl::Span<void*> batch);
7980

8081
class StaticForwarder {
8182
public:
82-
static HookList<InsertRangeHook> insert_range_hooks;
83-
static HookList<RemoveRangeHook> remove_range_hooks;
84-
8583
static void InvokeInsertRangeHook(size_t size_class,
8684
absl::Span<void*> batch) {
87-
if (ABSL_PREDICT_TRUE(insert_range_hooks.empty())) {
85+
if (ABSL_PREDICT_TRUE(central_freelist_insert_range_hooks.empty())) {
8886
return;
8987
}
9088
InvokeInsertRangeHookSlow(size_class, batch);
9189
}
9290

9391
static void InvokeRemoveRangeHook(size_t size_class,
9492
absl::Span<void*> batch) {
95-
if (ABSL_PREDICT_TRUE(remove_range_hooks.empty())) {
93+
if (ABSL_PREDICT_TRUE(central_freelist_remove_range_hooks.empty())) {
9694
return;
9795
}
9896
InvokeRemoveRangeHookSlow(size_class, batch);

‎tcmalloc/central_freelist_test.cc‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -831,8 +831,8 @@ TEST_P(CentralFreeListTest, HookTracing) {
831831
remove_count += batch.size();
832832
};
833833

834-
EXPECT_TRUE(e.forwarder().insert_range_hooks.Add(insert_hook));
835-
EXPECT_TRUE(e.forwarder().remove_range_hooks.Add(remove_hook));
834+
EXPECT_TRUE(e.forwarder().insert_range_hooks_.Add(insert_hook));
835+
EXPECT_TRUE(e.forwarder().remove_range_hooks_.Add(remove_hook));
836836

837837
void* batch[kMaxObjectsToMove];
838838
int got = e.central_freelist().RemoveRange(
@@ -843,8 +843,8 @@ TEST_P(CentralFreeListTest, HookTracing) {
843843
e.central_freelist().InsertRange({batch, static_cast<size_t>(got)});
844844
EXPECT_EQ(insert_count, got);
845845

846-
EXPECT_TRUE(e.forwarder().insert_range_hooks.Remove(insert_hook));
847-
EXPECT_TRUE(e.forwarder().remove_range_hooks.Remove(remove_hook));
846+
EXPECT_TRUE(e.forwarder().insert_range_hooks_.Remove(insert_hook));
847+
EXPECT_TRUE(e.forwarder().remove_range_hooks_.Remove(remove_hook));
848848
}
849849

850850
TEST_P(CentralFreeListTest, SpanLifetime) {

‎tcmalloc/internal/BUILD‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,22 @@ cc_library(
106106
],
107107
)
108108

109+
cc_library(
110+
name = "central_freelist_hooks",
111+
srcs = ["central_freelist_hooks.cc"],
112+
hdrs = ["central_freelist_hooks.h"],
113+
copts = TCMALLOC_DEFAULT_COPTS,
114+
visibility = ["//tcmalloc:__subpackages__"],
115+
deps = [
116+
":config",
117+
":hook_list",
118+
":logging",
119+
"@com_google_absl//absl/base",
120+
"@com_google_absl//absl/base:core_headers",
121+
"@com_google_absl//absl/types:span",
122+
],
123+
)
124+
109125
cc_library(
110126
name = "clock",
111127
hdrs = ["clock.h"],
@@ -697,6 +713,7 @@ cc_library(
697713
deps = [
698714
"//tcmalloc:malloc_extension",
699715
"@com_google_absl//absl/base:core_headers",
716+
"@com_google_absl//absl/base:nullability",
700717
"@com_google_absl//absl/time",
701718
],
702719
)

‎tcmalloc/internal/CMakeLists.txt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,24 @@ tcmalloc_cc_library(
100100
"tcmalloc::internal_logging"
101101
)
102102

103+
tcmalloc_cc_library(
104+
NAME
105+
tcmalloc_internal_central_freelist_hooks
106+
ALIAS
107+
tcmalloc::internal_central_freelist_hooks
108+
HDRS
109+
"central_freelist_hooks.h"
110+
SRCS
111+
"central_freelist_hooks.cc"
112+
DEPS
113+
"absl::base"
114+
"absl::core_headers"
115+
"absl::span"
116+
"tcmalloc::internal_config"
117+
"tcmalloc::internal_hook_list"
118+
"tcmalloc::internal_logging"
119+
)
120+
103121
tcmalloc_cc_library(
104122
NAME
105123
tcmalloc_internal_clock
@@ -705,6 +723,7 @@ tcmalloc_cc_library(
705723
"parameter_accessors.h"
706724
DEPS
707725
"absl::core_headers"
726+
"absl::nullability"
708727
"absl::time"
709728
"tcmalloc::malloc_extension"
710729
)

‎tcmalloc/internal/central_freelist_hooks.cc‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Copyright 2026 The TCMalloc Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "tcmalloc/internal/central_freelist_hooks.h"
16+
17+
#include <cstddef>
18+
19+
#include "absl/base/attributes.h"
20+
#include "absl/base/call_once.h"
21+
#include "absl/base/optimization.h"
22+
#include "absl/types/span.h"
23+
#include "tcmalloc/internal/config.h"
24+
#include "tcmalloc/internal/hook_list.h"
25+
#include "tcmalloc/internal/logging.h"
26+
27+
extern "C" {
28+
ABSL_ATTRIBUTE_WEAK void
29+
TCMalloc_CentralFreeList_InitAtFirstRemoveRange_Tracing();
30+
}
31+
32+
GOOGLE_MALLOC_SECTION_BEGIN
33+
namespace tcmalloc {
34+
namespace tcmalloc_internal {
35+
36+
namespace {
37+
38+
void InitialRemoveRangeHook(size_t size_class, absl::Span<void*> batch);
39+
40+
void RemoveInitialHooksAndCallInitializers() {
41+
TC_CHECK(central_freelist_remove_range_hooks.Remove(&InitialRemoveRangeHook));
42+
if (TCMalloc_CentralFreeList_InitAtFirstRemoveRange_Tracing != nullptr) {
43+
TCMalloc_CentralFreeList_InitAtFirstRemoveRange_Tracing();
44+
}
45+
}
46+
47+
void InitialRemoveRangeHook(size_t size_class, absl::Span<void*> batch) {
48+
ABSL_CONST_INIT static absl::once_flag once;
49+
absl::base_internal::LowLevelCallOnce(&once,
50+
RemoveInitialHooksAndCallInitializers);
51+
52+
if (ABSL_PREDICT_FALSE(!central_freelist_remove_range_hooks.empty())) {
53+
central_freelist_remove_range_hooks.Invoke(size_class, batch);
54+
}
55+
}
56+
57+
} // namespace
58+
59+
ABSL_CONST_INIT HookList<CentralFreelistInsertRangeHook>
60+
central_freelist_insert_range_hooks;
61+
ABSL_CONST_INIT HookList<CentralFreelistRemoveRangeHook>
62+
central_freelist_remove_range_hooks{&InitialRemoveRangeHook};
63+
64+
} // namespace tcmalloc_internal
65+
} // namespace tcmalloc
66+
GOOGLE_MALLOC_SECTION_END

0 commit comments

Comments
 (0)