|
6 | 6 |
|
7 | 7 | #include "callTraceStorage.h" |
8 | 8 | #include "counters.h" |
| 9 | +#include "log.h" |
9 | 10 | #include "os.h" |
10 | 11 | #include "common.h" |
11 | 12 | #include "thread.h" |
12 | 13 | #include "vmEntry.h" // For BCI_ERROR constant |
13 | 14 | #include "arch.h" // For LP64_ONLY macro and COMMA macro |
14 | 15 | #include "guards.h" // For table swap critical sections |
15 | | -#include "primeProbing.h" |
16 | 16 | #include "thread.h" |
17 | 17 | #include <string.h> |
18 | 18 | #include <atomic> |
19 | | -#include <time.h> |
20 | | - |
21 | | -// RefCountGuard static members |
22 | | -RefCountSlot RefCountGuard::refcount_slots[RefCountGuard::MAX_THREADS]; |
23 | | -int RefCountGuard::slot_owners[RefCountGuard::MAX_THREADS]; |
24 | | - |
25 | | - |
26 | | -// RefCountGuard implementation |
27 | | -int RefCountGuard::getThreadRefCountSlot() { |
28 | | - // Signal-safe collision resolution: use OS::threadId() with semi-random prime step probing |
29 | | - ProfiledThread* thrd = ProfiledThread::currentSignalSafe(); |
30 | | - int tid = thrd != nullptr ? thrd->tid() : OS::threadId(); |
31 | | - |
32 | | - // Semi-random prime step probing to eliminate secondary clustering |
33 | | - HashProbe probe(static_cast<u64>(tid), MAX_THREADS); |
34 | | - |
35 | | - int slot = probe.slot(); |
36 | | - for (int i = 0; i < MAX_PROBE_DISTANCE; i++) { |
37 | | - // Try to claim this slot atomically |
38 | | - int expected = 0; // Empty slot (no thread ID) |
39 | | - if (__atomic_compare_exchange_n(&slot_owners[slot], &expected, tid, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { |
40 | | - // Successfully claimed the slot |
41 | | - return slot; |
42 | | - } |
43 | | - |
44 | | - // Check if we already own this slot (for reentrant calls) |
45 | | - if (__atomic_load_n(&slot_owners[slot], __ATOMIC_ACQUIRE) == tid) { |
46 | | - return slot; |
47 | | - } |
48 | | - |
49 | | - // Move to next slot using probe |
50 | | - if (probe.hasNext()) { |
51 | | - slot = probe.next(); |
52 | | - } |
53 | | - } |
54 | | - |
55 | | - // All probing attempts failed - return -1 to indicate failure |
56 | | - return -1; |
57 | | -} |
58 | | - |
59 | | -RefCountGuard::RefCountGuard(CallTraceHashTable* resource) : _active(true), _my_slot(-1) { |
60 | | - // Get thread refcount slot using signal-safe collision resolution |
61 | | - _my_slot = getThreadRefCountSlot(); |
62 | | - |
63 | | - if (_my_slot == -1) { |
64 | | - // Slot allocation failed - refcount guard is inactive |
65 | | - _active = false; |
66 | | - return; |
67 | | - } |
68 | | - |
69 | | - // CRITICAL ORDERING: Store pointer FIRST, then increment count |
70 | | - // This ensures the pointer-first protocol for race-free operation |
71 | | - // |
72 | | - // Why this ordering is safe: |
73 | | - // Between step 1 and 2, if scanner runs: |
74 | | - // - Scanner loads count=0 (not yet incremented) |
75 | | - // - Scanner sees slot as inactive, skips it |
76 | | - // - Safe: we haven't "activated" protection yet |
77 | | - // |
78 | | - // After step 2, slot is fully active and protects the resource |
79 | | - __atomic_store_n(&refcount_slots[_my_slot].active_table, resource, __ATOMIC_RELEASE); |
80 | | - __atomic_fetch_add(&refcount_slots[_my_slot].count, 1, __ATOMIC_RELEASE); |
81 | | -} |
82 | | - |
83 | | -RefCountGuard::~RefCountGuard() { |
84 | | - if (_active && _my_slot >= 0) { |
85 | | - // CRITICAL ORDERING: Decrement count FIRST, then clear pointer |
86 | | - // This ensures safe deactivation |
87 | | - // |
88 | | - // Why this ordering is safe: |
89 | | - // After step 1, count=0 so scanner will skip this slot |
90 | | - // Step 2 clears the pointer (cleanup) |
91 | | - // No window where scanner thinks slot protects a table it doesn't |
92 | | - __atomic_fetch_sub(&refcount_slots[_my_slot].count, 1, __ATOMIC_RELEASE); |
93 | | - __atomic_store_n(&refcount_slots[_my_slot].active_table, nullptr, __ATOMIC_RELEASE); |
94 | | - |
95 | | - // Release slot ownership |
96 | | - __atomic_store_n(&slot_owners[_my_slot], 0, __ATOMIC_RELEASE); |
97 | | - } |
98 | | -} |
99 | | - |
100 | | -RefCountGuard::RefCountGuard(RefCountGuard&& other) noexcept : _active(other._active), _my_slot(other._my_slot) { |
101 | | - other._active = false; |
102 | | -} |
103 | | - |
104 | | -RefCountGuard& RefCountGuard::operator=(RefCountGuard&& other) noexcept { |
105 | | - if (this != &other) { |
106 | | - // Clean up current state with same ordering as destructor |
107 | | - if (_active && _my_slot >= 0) { |
108 | | - __atomic_fetch_sub(&refcount_slots[_my_slot].count, 1, __ATOMIC_RELEASE); |
109 | | - __atomic_store_n(&refcount_slots[_my_slot].active_table, nullptr, __ATOMIC_RELEASE); |
110 | | - __atomic_store_n(&slot_owners[_my_slot], 0, __ATOMIC_RELEASE); |
111 | | - } |
112 | | - |
113 | | - // Move from other |
114 | | - _active = other._active; |
115 | | - _my_slot = other._my_slot; |
116 | | - |
117 | | - // Clear other |
118 | | - other._active = false; |
119 | | - } |
120 | | - return *this; |
121 | | -} |
122 | | - |
123 | | -void RefCountGuard::waitForRefCountToClear(CallTraceHashTable* table_to_delete) { |
124 | | - // Check refcount slots for the table we want to delete |
125 | | - // |
126 | | - // POINTER-FIRST PROTOCOL GUARANTEES: |
127 | | - // - Constructor stores pointer then increments count |
128 | | - // - Destructor decrements count then clears pointer |
129 | | - // - Scanner checks count first (if 0, slot is inactive) |
130 | | - // |
131 | | - // TRACE DROP WINDOW (intentional design): |
132 | | - // - Scanner can complete on FIRST iteration if all slots have count=0 |
133 | | - // - Guards in construction (pointer stored, count still 0) are treated as inactive |
134 | | - // - Revalidation check in put() detects this race and drops the trace |
135 | | - // - This trades a narrow trace-drop window (~10-100ns) for protocol simplicity |
136 | | - // - USE-AFTER-FREE IS IMPOSSIBLE: Revalidation prevents table access after deletion |
137 | | - |
138 | | - // PHASE 1: Fast path - spin with pause for short waits (common case) |
139 | | - // Expected: refcounts clear within 1-20µs as put() operations complete |
140 | | - const int SPIN_ITERATIONS = 100; |
141 | | - for (int spin = 0; spin < SPIN_ITERATIONS; ++spin) { |
142 | | - bool all_clear = true; |
143 | | - |
144 | | - // Scan all slots (no bitmap optimization, but simpler logic) |
145 | | - for (int i = 0; i < MAX_THREADS; ++i) { |
146 | | - // CRITICAL: Check count FIRST (pointer-first protocol) |
147 | | - uint32_t count = __atomic_load_n(&refcount_slots[i].count, __ATOMIC_ACQUIRE); |
148 | | - if (count == 0) { |
149 | | - continue; // Slot inactive, skip it |
150 | | - } |
151 | | - |
152 | | - // Count > 0, so slot is active - check which table it protects |
153 | | - CallTraceHashTable* table = __atomic_load_n(&refcount_slots[i].active_table, __ATOMIC_ACQUIRE); |
154 | | - if (table == table_to_delete) { |
155 | | - all_clear = false; |
156 | | - break; |
157 | | - } |
158 | | - } |
159 | | - |
160 | | - if (all_clear) { |
161 | | - return; // Fast path success - refcounts cleared quickly |
162 | | - } |
163 | | - spinPause(); // CPU pause instruction, ~10-50 cycles |
164 | | - } |
165 | | - |
166 | | - // PHASE 2: Slow path - async-signal-safe sleep for blocked thread case |
167 | | - const int MAX_WAIT_ITERATIONS = 5000; |
168 | | - struct timespec sleep_time = {0, 100000}; // 100 microseconds |
169 | | - |
170 | | - for (int wait_count = 0; wait_count < MAX_WAIT_ITERATIONS; ++wait_count) { |
171 | | - bool all_clear = true; |
172 | | - |
173 | | - for (int i = 0; i < MAX_THREADS; ++i) { |
174 | | - uint32_t count = __atomic_load_n(&refcount_slots[i].count, __ATOMIC_ACQUIRE); |
175 | | - if (count == 0) { |
176 | | - continue; |
177 | | - } |
178 | | - |
179 | | - CallTraceHashTable* table = __atomic_load_n(&refcount_slots[i].active_table, __ATOMIC_ACQUIRE); |
180 | | - if (table == table_to_delete) { |
181 | | - all_clear = false; |
182 | | - break; |
183 | | - } |
184 | | - } |
185 | | - |
186 | | - if (all_clear) { |
187 | | - return; // Slow path success |
188 | | - } |
189 | | - |
190 | | - // nanosleep is POSIX async-signal-safe and does not call malloc |
191 | | - nanosleep(&sleep_time, nullptr); |
192 | | - } |
193 | | - |
194 | | - // If we reach here, some refcounts didn't clear in time |
195 | | - // This shouldn't happen in normal operation but we log it for debugging |
196 | | -} |
197 | | - |
198 | | -void RefCountGuard::waitForAllRefCountsToClear() { |
199 | | - // PHASE 1: Fast path - spin with pause for short waits |
200 | | - const int SPIN_ITERATIONS = 100; |
201 | | - for (int spin = 0; spin < SPIN_ITERATIONS; ++spin) { |
202 | | - bool any_refcounts = false; |
203 | | - |
204 | | - for (int i = 0; i < MAX_THREADS; ++i) { |
205 | | - uint32_t count = __atomic_load_n(&refcount_slots[i].count, __ATOMIC_ACQUIRE); |
206 | | - if (count > 0) { |
207 | | - any_refcounts = true; |
208 | | - break; |
209 | | - } |
210 | | - } |
211 | | - |
212 | | - if (!any_refcounts) { |
213 | | - return; // Fast path success |
214 | | - } |
215 | | - spinPause(); |
216 | | - } |
217 | | - |
218 | | - // PHASE 2: Slow path - async-signal-safe sleep |
219 | | - const int MAX_WAIT_ITERATIONS = 5000; |
220 | | - struct timespec sleep_time = {0, 100000}; // 100 microseconds |
221 | | - |
222 | | - for (int wait_count = 0; wait_count < MAX_WAIT_ITERATIONS; ++wait_count) { |
223 | | - bool any_refcounts = false; |
224 | | - |
225 | | - for (int i = 0; i < MAX_THREADS; ++i) { |
226 | | - uint32_t count = __atomic_load_n(&refcount_slots[i].count, __ATOMIC_ACQUIRE); |
227 | | - if (count > 0) { |
228 | | - any_refcounts = true; |
229 | | - break; |
230 | | - } |
231 | | - } |
232 | | - |
233 | | - if (!any_refcounts) { |
234 | | - return; // Slow path success |
235 | | - } |
236 | | - |
237 | | - nanosleep(&sleep_time, nullptr); |
238 | | - } |
239 | | - |
240 | | - // If we reach here, some refcounts didn't clear in time |
241 | | -} |
242 | | - |
243 | 19 |
|
244 | 20 | static const u64 OVERFLOW_TRACE_ID = 0x7fffffffffffffffULL; // Max 64-bit signed value |
245 | 21 |
|
|
0 commit comments