Improve accuracy of alloc metrics with freelists (#12268)

Author: cmcfarlen

include/iocore/eventsystem/ProxyAllocator.h

@@ -57,6 +57,9 @@ thread_alloc(CAlloc &a, ProxyAllocator &l, Args &&...args)
     void *v    = l.freelist;
     l.freelist = *reinterpret_cast<void **>(l.freelist);
     --(l.allocated);
+#if TS_USE_ALLOCATOR_METRICS
+    a.increment_for_alloc();
+#endif
     ::new (v) typename CAlloc::Value_type(std::forward<Args>(args)...);
     return static_cast<typename CAlloc::Value_type *>(v);
   }
@@ -84,6 +87,12 @@ void thread_freeup(Allocator &a, ProxyAllocator &l);
 
 #endif
 
+#if TS_USE_ALLOCATOR_METRICS
+#define UPDATE_FREE_METRICS(_a) _a.increment_for_free()
+#else
+#define UPDATE_FREE_METRICS(_a)
+#endif
+
 #define THREAD_FREE(_p, _a, _tin)                                                                  \
   do {                                                                                             \
     ::_a.destroy_if_enabled(_p);                                                                   \
@@ -92,6 +101,7 @@ void thread_freeup(Allocator &a, ProxyAllocator &l);
       *(char **)_p    = (char *)_t->_a.freelist;                                                   \
       _t->_a.freelist = _p;                                                                        \
       _t->_a.allocated++;                                                                          \
+      UPDATE_FREE_METRICS(::_a);                                                                   \
       if (thread_freelist_high_watermark > 0 && _t->_a.allocated > thread_freelist_high_watermark) \
         thread_freeup(::_a.raw(), _t->_a);                                                         \
     } else {                                                                                       \

include/tscore/Allocator.h

@@ -227,27 +227,43 @@ template <typename WrappedAllocator> class MeteredAllocator : public WrappedAllo
   void *
   alloc_void()
   {
-    inuse_metric->increment(1);
-    alloc_metric->increment(1);
+    increment_for_alloc();
     return WrappedAllocator::alloc_void();
   }
 
   void
   free_void(void *ptr)
   {
-    inuse_metric->decrement(1);
-    free_metric->increment(1);
+    increment_for_free();
     WrappedAllocator::free_void(ptr);
   }
 
   void
   free_void_bulk(void *head, void *tail, size_t num_item)
   {
-    inuse_metric->decrement(num_item);
-    free_metric->increment(num_item);
+    // this is only currently called from ProxyAllocator to free
+    // back to the watermark, so if this is called, it means
+    // the ProxyAllocator free has been decrementing the free count
+    // for items on its freelist so this function should not update metrics
     WrappedAllocator::free_void_bulk(head, tail, num_item);
   }
 
+  // NOTE(cmcfarlen): These metric functions are also called
+  // by ProxyAllocator when items are added/removed from its freelist
+  void
+  increment_for_alloc()
+  {
+    inuse_metric->increment(1);
+    alloc_metric->increment(1);
+  }
+
+  void
+  increment_for_free(int val = 1)
+  {
+    inuse_metric->decrement(val);
+    free_metric->increment(val);
+  }
+
   MeteredAllocator() {}
 
   MeteredAllocator(const char *name, unsigned int element_size, unsigned int chunk_size = 128, unsigned int alignment = 8,