Add likely/unlikely branch prediction hints to hot-path subsystems

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

Author: jbachorik

ddprof-lib/src/main/cpp/buffers.h

@@ -42,7 +42,7 @@ class Buffer {
   virtual int limit() const { return _limit; }
 
   bool flushIfNeeded(FlushCallback callback, int limit = BUFFER_LIMIT) {
-    if (_offset > limit) {
+    if (unlikely(_offset > limit)) {
       if (callback(_data, _offset) == _offset) {
         reset();
         return true;

ddprof-lib/src/main/cpp/callTraceHashTable.cpp

@@ -48,7 +48,7 @@ class LongHashTable {
 
   static LongHashTable *allocate(LongHashTable *prev, u32 capacity, LinearAllocator* allocator) {
     void *memory = allocator->alloc(getSize(capacity));
-    if (memory != nullptr) {
+    if (likely(memory != nullptr)) {
       // Use placement new to invoke constructor in-place with parameters
       // LinearAllocator doesn't zero memory like OS::safeAlloc with anon mmap
       // so we need to explicitly clear the keys and values (should_clean = true)
@@ -174,7 +174,7 @@ CallTrace *CallTraceHashTable::storeCallTrace(int num_frames,
   const size_t total_size = header_size + num_frames * sizeof(ASGCT_CallFrame);
   void *memory = _allocator.alloc(total_size);
   CallTrace *buf = nullptr;
-  if (memory != nullptr) {
+  if (likely(memory != nullptr)) {
     // Use placement new to invoke constructor in-place
     buf = new (memory) CallTrace(truncated, num_frames, trace_id);
     // Do not use memcpy inside signal handler
@@ -199,7 +199,7 @@ CallTrace *CallTraceHashTable::findCallTrace(LongHashTable *table, u64 hash) {
     if (keys[slot] == 0) {
       return nullptr;
     }
-    if (!probe.hasNext()) {
+    if (unlikely(!probe.hasNext())) {
       break;
     }
     slot = probe.next();
@@ -213,7 +213,7 @@ u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
   u64 hash = calcHash(num_frames, frames, truncated);
 
   LongHashTable *table = _table;
-  if (table == nullptr) {
+  if (unlikely(table == nullptr)) {
     // Table allocation failed or was cleared - drop sample
     Counters::increment(CALLTRACE_STORAGE_DROPPED);
     return CallTraceStorage::DROPPED_TRACE_ID;
@@ -230,7 +230,7 @@ u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
       CallTrace* current_trace = table->values()[slot].acquireTrace();
 
       // If another thread is preparing this slot, wait for completion
-      if (current_trace == CallTraceSample::PREPARING) {
+      if (unlikely(current_trace == CallTraceSample::PREPARING)) {
         // Wait for the preparing thread to complete, with timeout
         int wait_cycles = 0;
         const int MAX_WAIT_CYCLES = 1000; // ~1000 cycles should be enough for allocation
@@ -252,13 +252,13 @@ u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
         } while (current_trace == CallTraceSample::PREPARING && wait_cycles < MAX_WAIT_CYCLES);
 
         // If still preparing after timeout, something is wrong - continue search
-        if (current_trace == CallTraceSample::PREPARING) {
+        if (unlikely(current_trace == CallTraceSample::PREPARING)) {
           continue;
         }
       }
 
       // Check final state after waiting
-      if (current_trace != nullptr && current_trace != CallTraceSample::PREPARING) {
+      if (likely(current_trace != nullptr && current_trace != CallTraceSample::PREPARING)) {
         // Trace is ready, use it
         return current_trace->trace_id;
       } else {
@@ -279,7 +279,7 @@ u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
       }
 
       // Mark the slot as being prepared so other threads know to wait
-      if (!table->values()[slot].markPreparing()) {
+      if (unlikely(!table->values()[slot].markPreparing())) {
         // Failed to mark as preparing (shouldn't happen), clear key with full barrier and retry
         __atomic_thread_fence(__ATOMIC_SEQ_CST);
         __atomic_store_n(&keys[slot], 0, __ATOMIC_RELEASE);
@@ -293,10 +293,10 @@ u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
       probe.updateCapacity(new_size);
 
       // EXPANSION LOGIC: Check if 75% capacity reached after incrementing size
-      if (new_size == capacity * 3 / 4) {
+      if (unlikely(new_size == capacity * 3 / 4)) {
         // Allocate new table with double capacity using LinearAllocator
         LongHashTable* new_table = LongHashTable::allocate(table, capacity * 2, &_allocator);
-        if (new_table != nullptr) {
+        if (likely(new_table != nullptr)) {
           // Atomic table swap - only one thread succeeds
           __atomic_compare_exchange_n(&_table, &table, new_table, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
         }
@@ -315,7 +315,7 @@ u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
         u64 instance_id = _instance_id.load(std::memory_order_acquire);
         u64 trace_id = (instance_id << 32) | slot;
         trace = storeCallTrace(num_frames, frames, truncated, trace_id);
-        if (trace == nullptr) {
+        if (unlikely(trace == nullptr)) {
           // Allocation failure - reset trace first, then clear key
           table->values()[slot].setTrace(nullptr);
           __atomic_thread_fence(__ATOMIC_SEQ_CST);
@@ -330,7 +330,7 @@ u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
       return trace->trace_id;
     }
 
-    if (!probe.hasNext()) {
+    if (unlikely(!probe.hasNext())) {
       // Table overflow - very unlikely with expansion logic
       atomicIncRelaxed(_overflow);
       return OVERFLOW_TRACE_ID;

ddprof-lib/src/main/cpp/callTraceStorage.cpp

@@ -345,7 +345,7 @@ u64 CallTraceStorage::put(int num_frames, ASGCT_CallFrame* frames, bool truncate
     CallTraceHashTable* active = const_cast<CallTraceHashTable*>(__atomic_load_n(&_active_storage, __ATOMIC_ACQUIRE));
 
     // Safety check - if null, system is shutting down
-    if (active == nullptr) {
+    if (unlikely(active == nullptr)) {
         Counters::increment(CALLTRACE_STORAGE_DROPPED);
         return DROPPED_TRACE_ID;
     }
@@ -355,7 +355,7 @@ u64 CallTraceStorage::put(int num_frames, ASGCT_CallFrame* frames, bool truncate
     RefCountGuard guard(active);
 
     // Check if refcount guard allocation failed (slot exhaustion)
-    if (!guard.isActive()) {
+    if (unlikely(!guard.isActive())) {
         // No refcount protection available - return dropped trace ID
         Counters::increment(CALLTRACE_STORAGE_DROPPED);
         return DROPPED_TRACE_ID;
@@ -373,7 +373,7 @@ u64 CallTraceStorage::put(int num_frames, ASGCT_CallFrame* frames, bool truncate
     //
     // Memory ordering: ACQUIRE load ensures we see scanner's ACQ_REL exchange to nullptr
     CallTraceHashTable* original_active = const_cast<CallTraceHashTable*>(__atomic_load_n(&_active_storage, __ATOMIC_ACQUIRE));
-    if (original_active != active || original_active == nullptr) {
+    if (unlikely(original_active != active || original_active == nullptr)) {
         // Storage was swapped or nullified during guard construction
         // SAFE: We detected the race, drop this trace, never use the table pointer
         Counters::increment(CALLTRACE_STORAGE_DROPPED);

ddprof-lib/src/main/cpp/codeCache.cpp

@@ -25,11 +25,11 @@ char *NativeFunc::create(const char *name, short lib_index) {
 void NativeFunc::destroy(char *name) { free(from(name)); }
 
 char NativeFunc::read_mark(const char* name) {
-  if (name == nullptr) {
+  if (unlikely(name == nullptr)) {
     return 0;
   }
   NativeFunc* func = from(name);
-  if (!is_aligned(func, sizeof(func))) {
+  if (unlikely(!is_aligned(func, sizeof(func)))) {
     return 0;
   }
   // Use SafeAccess to read the mark field in signal handler context
@@ -394,7 +394,7 @@ void CodeCache::setDwarfTable(FrameDesc *table, int length) {
 }
 
 FrameDesc CodeCache::findFrameDesc(const void *pc) {
-  if (_dwarf_table == NULL || _dwarf_table_length == 0) {
+  if (unlikely(_dwarf_table == NULL || _dwarf_table_length == 0)) {
     // No DWARF data available - use default frame pointer unwinding
     // This handles OpenJ9 and other VMs that don't provide DWARF info
     return FrameDesc::default_frame;
@@ -415,7 +415,7 @@ FrameDesc CodeCache::findFrameDesc(const void *pc) {
     }
   }
 
-  if (low > 0) {
+  if (likely(low > 0)) {
     return _dwarf_table[low - 1];
   } else if (target_loc - _plt_offset < _plt_size) {
     return FrameDesc::empty_frame;

ddprof-lib/src/main/cpp/context.cpp

@@ -40,7 +40,7 @@ Context& Contexts::initializeContextTls() {
 
 Context& Contexts::get() {
   ProfiledThread* thrd = ProfiledThread::currentSignalSafe();
-  if (thrd == nullptr || !thrd->isContextTlsInitialized()) {
+  if (unlikely(thrd == nullptr || !thrd->isContextTlsInitialized())) {
     return DD_EMPTY_CONTEXT;
   }
   // Return via stored pointer - never access context_tls_v1 from signal handler

ddprof-lib/src/main/cpp/flightRecorder.cpp

@@ -794,7 +794,7 @@ void Recording::flush(Buffer *buf) {
 }
 
 void Recording::flushIfNeeded(Buffer *buf, int limit) {
-  if (buf->offset() >= limit) {
+  if (unlikely(buf->offset() >= limit)) {
     flush(buf);
   }
 }

ddprof-lib/src/main/cpp/linearAllocator.cpp

@@ -138,7 +138,7 @@ void *LinearAllocator::alloc(size_t size) {
 
   // CRITICAL FIX: After detachChunks() fails, _tail may be nullptr.
   // We must handle this gracefully to prevent crash.
-  if (chunk == nullptr) {
+  if (unlikely(chunk == nullptr)) {
     return nullptr;
   }
 
@@ -157,7 +157,7 @@ void *LinearAllocator::alloc(size_t size) {
         ASAN_UNPOISON_MEMORY_REGION(allocated_ptr, size);
         #endif
 
-        if (_chunk_size / 2 - offs < size) {
+        if (unlikely(_chunk_size / 2 - offs < size)) {
           // Stepped over a middle of the chunk - it's time to prepare a new one
           reserveChunk(chunk);
         }
@@ -171,7 +171,7 @@ void *LinearAllocator::alloc(size_t size) {
 
 Chunk *LinearAllocator::allocateChunk(Chunk *current) {
   Chunk *chunk = (Chunk *)OS::safeAlloc(_chunk_size);
-  if (chunk != NULL) {
+  if (likely(chunk != NULL)) {
     chunk->prev = current;
     chunk->offs = sizeof(Chunk);
 
@@ -208,18 +208,18 @@ void LinearAllocator::reserveChunk(Chunk *current) {
 Chunk *LinearAllocator::getNextChunk(Chunk *current) {
   Chunk *reserve = _reserve;
 
-  if (reserve == current) {
+  if (unlikely(reserve == current)) {
     // Unlikely case: no reserve yet.
     // It's probably being allocated right now, so let's compete
     reserve = allocateChunk(current);
-    if (reserve == NULL) {
+    if (unlikely(reserve == NULL)) {
       // Not enough memory
       return NULL;
     }
 
     Chunk *prev_reserve =
         __sync_val_compare_and_swap(&_reserve, current, reserve);
-    if (prev_reserve != current) {
+    if (unlikely(prev_reserve != current)) {
       freeChunk(reserve);
       reserve = prev_reserve;
     }

ddprof-lib/src/main/cpp/profiler.cpp

@@ -764,9 +764,9 @@ u64 Profiler::recordJVMTISample(u64 counter, int tid, jthread thread, jint event
   atomicIncRelaxed(_total_samples);
 
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     // Too many concurrent signals already
     atomicIncRelaxed(_failures[-ticks_skipped]);
 
@@ -811,9 +811,9 @@ void Profiler::recordDeferredSample(int tid, u64 call_trace_id, jint event_type,
   atomicIncRelaxed(_total_samples);
 
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     // Too many concurrent signals already
     atomicIncRelaxed(_failures[-ticks_skipped]);
     return;
@@ -829,9 +829,9 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
   atomicIncRelaxed(_total_samples);
 
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     // Too many concurrent signals already
     atomicIncRelaxed(_failures[-ticks_skipped]);
 
@@ -849,7 +849,7 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
   // record a null stacktrace we can skip the unwind if we've got a
   // call_trace_id determined to be reusable at a higher level
 
-  if (!_omit_stacktraces && call_trace_id == 0) {
+  if (likely(!_omit_stacktraces && call_trace_id == 0)) {
     u64 startTime = TSC::ticks();
     ASGCT_CallFrame *frames = _calltrace_buffer[lock_index]->_asgct_frames;
 
@@ -859,7 +859,7 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
     ASGCT_CallFrame *native_stop = frames + num_frames;
     num_frames += getNativeTrace(ucontext, native_stop, event_type, tid,
                                  &java_ctx, &truncated, lock_index);
-    if (num_frames < _max_stack_depth) {
+    if (likely(num_frames < _max_stack_depth)) {
       int max_remaining = _max_stack_depth - num_frames;
       if (_features.mixed) {
         int vm_start = num_frames;
@@ -887,14 +887,14 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
         }
       }
     }
-    if (num_frames == 0) {
+    if (unlikely(num_frames == 0)) {
       num_frames += makeFrame(frames + num_frames, BCI_ERROR, "no_Java_frame");
     }
 
     call_trace_id =
         _call_trace_storage.put(num_frames, frames, truncated, counter);
     ProfiledThread *thread = ProfiledThread::currentSignalSafe();
-    if (thread != nullptr) {
+    if (likely(thread != nullptr)) {
       thread->recordCallTraceId(call_trace_id);
     }
     u64 duration = TSC::ticks() - startTime;
@@ -909,9 +909,9 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid,
 
 void Profiler::recordWallClockEpoch(int tid, WallClockEpochEvent *event) {
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.wallClockEpoch(lock_index, event);
@@ -920,9 +920,9 @@ void Profiler::recordWallClockEpoch(int tid, WallClockEpochEvent *event) {
 
 void Profiler::recordTraceRoot(int tid, TraceRootEvent *event) {
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.recordTraceRoot(lock_index, tid, event);
@@ -931,9 +931,9 @@ void Profiler::recordTraceRoot(int tid, TraceRootEvent *event) {
 
 void Profiler::recordQueueTime(int tid, QueueTimeEvent *event) {
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.recordQueueTime(lock_index, tid, event);
@@ -957,9 +957,9 @@ void Profiler::recordExternalSample(u64 weight, int tid, int num_frames,
 
   u64 call_trace_id =
       _call_trace_storage.put(num_frames, extended_frames, truncated, weight);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     // Too many concurrent signals already
     atomicIncRelaxed(_failures[-ticks_skipped]);
     return;
@@ -982,9 +982,9 @@ void Profiler::writeDatadogProfilerSetting(int tid, int length,
                                            const char *name, const char *value,
                                            const char *unit) {
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.recordDatadogSetting(lock_index, length, name, value, unit);
@@ -997,9 +997,9 @@ void Profiler::writeHeapUsage(long value, bool live) {
     return;
   }
   u32 lock_index = getLockIndex(tid);
-  if (!_locks[lock_index].tryLock() &&
+  if (unlikely(!_locks[lock_index].tryLock() &&
       !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) {
+      !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())) {
     return;
   }
   _jfr.recordHeapUsage(lock_index, value, live);