Feature: support using gdb to debug fiber (#64)

Author: chhy2009

docs/en/fiber_user_guide.md

@@ -342,6 +342,10 @@ trpc::StartFiberDetached([&] {
 
 ```
 
+# Debugging fibers using gdb
+
+See [gdb_plugin](../../trpc/tools/gdb_plugin/)
+
 # FAQ
 
 See more [Fiber FAQ](./fiber_faq.md)

docs/zh/fiber_user_guide.md

@@ -342,6 +342,10 @@ trpc::StartFiberDetached([&] {
 
 ```
 
+# 使用gdb调试fiber
+
+查阅[gdb_plugin](../../trpc/tools/gdb_plugin/)
+
 # FAQ
 
 查阅[Fiber FAQ](./faq/fiber_problem.md)

trpc/BUILD

@@ -108,3 +108,4 @@ config_setting(
     name = "trpc_enable_http_transinfo_base64",
     values = {"define": "trpc_enable_http_transinfo_base64=true"},
 )
+

trpc/common/config/global_conf.cc

@@ -65,6 +65,7 @@ void FiberThreadModelInstanceConfig::Display() const {
   TRPC_LOG_DEBUG("fiber_pool_num_by_mmap:" << fiber_pool_num_by_mmap);
   TRPC_LOG_DEBUG("fiber_stack_enable_guard_page:" << fiber_stack_enable_guard_page);
   TRPC_LOG_DEBUG("fiber_scheduling_name:" << fiber_scheduling_name);
+  TRPC_LOG_DEBUG("enable_gdb_debug:" << enable_gdb_debug);
 
   TRPC_LOG_DEBUG("================================");
 }

trpc/common/config/global_conf.h

@@ -84,6 +84,9 @@ struct FiberThreadModelInstanceConfig {
   /// @brief Stack overflow protect
   bool fiber_stack_enable_guard_page{true};
 
+  /// @brief Enable debug fiber using gdb
+  bool enable_gdb_debug = false;
+
   void Display() const;
 };
 

trpc/common/config/global_conf_parser.h

@@ -43,6 +43,7 @@ struct convert<trpc::FiberThreadModelInstanceConfig> {
     node["fiber_pool_num_by_mmap"] = config.fiber_pool_num_by_mmap;
     node["fiber_stack_enable_guard_page"] = config.fiber_stack_enable_guard_page;
     node["fiber_scheduling_name"] = config.fiber_scheduling_name;
+    node["enable_gdb_debug"] = config.enable_gdb_debug;
 
     return node;
   }
@@ -112,6 +113,10 @@ struct convert<trpc::FiberThreadModelInstanceConfig> {
       config.fiber_scheduling_name = "v1";
     }
 
+    if (node["enable_gdb_debug"]) {
+      config.enable_gdb_debug = node["enable_gdb_debug"].as<bool>();
+    }
+
     return true;
   }
 };

trpc/runtime/fiber_runtime.cc

@@ -105,6 +105,7 @@ void StartRuntime() {
     options.pool_num_by_mmap = conf.fiber_pool_num_by_mmap;
     options.stack_enable_guard_page = conf.fiber_stack_enable_guard_page;
     options.disable_process_name = global_config.thread_disable_process_name;
+    options.enable_gdb_debug = conf.enable_gdb_debug;
   } else {
     options.group_name = "fiber_instance";
   }

trpc/runtime/threadmodel/fiber/detail/BUILD

@@ -26,6 +26,7 @@ cc_library(
     deps = [
         ":assembly",
         "//trpc/util:check",
+        "//trpc/util:deferred",
         "//trpc/util:likely",
         "//trpc/util/internal:never_destroyed",
     ],

trpc/runtime/threadmodel/fiber/detail/stack_allocator_impl.cc

@@ -9,6 +9,11 @@
 // please note that tRPC source code is licensed under the  Apache 2.0 License,
 // A copy of the Apache 2.0 License is included in this file.
 //
+// tRPC is licensed under the Apache 2.0 License, and includes source codes from
+// the following components:
+// 1. flare
+// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
+// flare is licensed under the BSD 3-Clause License.
 //
 
 #include "trpc/runtime/threadmodel/fiber/detail/stack_allocator_impl.h"
@@ -22,6 +27,7 @@
 
 #include "trpc/runtime/threadmodel/fiber/detail/assembly.h"
 #include "trpc/util/check.h"
+#include "trpc/util/deferred.h"
 #include "trpc/util/internal/never_destroyed.h"
 #include "trpc/util/likely.h"
 
@@ -31,13 +37,123 @@ static bool fiber_use_mmap = true;
 static uint32_t fiber_stack_size = 131072;
 static bool fiber_stack_enable_guard_page = true;
 static uint32_t max_fiber_num_by_mmap = 30 * 1024;
+static bool enable_gdb_debug = false;
+
+const uint32_t kPageSize = getpagesize();
+
+// The following source codes are from flare.
+// Copied and modified from
+// https://github.com/Tencent/flare/blob/master/flare/fiber/detail/stack_allocator.cc
+
+// All stacks (whether system stack or user stack) are registered here. This is
+// necessary for our GDB plugin to find all the stacks.
+//
+// Only _actual_ stack allocation / deallocation needs to touch this. For
+// allocations / deallocations covered by our object pool, they're irrelevant
+// here.
+//
+// Registration / deregistration can be slow. But that's okay as it's already
+// slow enough to _actually_ creating / destroying stacks. These operations
+// incur heavy VMA operations.
+struct StackRegistry {
+  // Listed as public as they're our "public" interfaces to GDB plugin.
+  //
+  // Code in this TU should use methods below instead of touching these fields.
+  void** stacks = nullptr;  // Leaked on exit. Doesn't matter.
+  std::size_t used = 0;
+  std::size_t capacity = 0;
+
+  // Register a newly-allocated stack.
+  //
+  // `ptr` should point to stack bottom (i.e. one byte past the stack region).
+  // That's where our fiber control block (GDB plugin need it) resides.
+  void RegisterStack(void* ptr) {
+    std::scoped_lock _(lock_);  // It's slow, so be it.
+    ++used;
+    auto slot = UnsafeFindSlotOf(nullptr);
+    if (slot) {
+      *slot = ptr;
+      return;
+    }
+
+    UnsafeResizeRegistry();
+    *UnsafeFindSlotOf(nullptr) = ptr;  // Must succeed this time.
+  }
+
+  // Deregister a going-to-be-freed stack. `ptr` points to stack bottom.
+  void DeregisterStack(void* ptr) {
+    std::scoped_lock _(lock_);
+
+    trpc::ScopedDeferred __([&] {
+      // If `stacks` is too large we should consider shrinking it.
+      if (capacity > 1024 && capacity / 2 > used) {
+        UnsafeShrinkRegistry();
+      }
+    });
+
+    --used;
+    if (auto p = UnsafeFindSlotOf(ptr)) {
+      *p = nullptr;
+      return;
+    }
+    TRPC_UNREACHABLE();
+  }
+
+ private:
+  void** UnsafeFindSlotOf(void* ptr) {
+    for (std::size_t i = 0; i != capacity; ++i) {
+      if (stacks[i] == ptr) {
+        return &stacks[i];
+      }
+    }
+    return nullptr;
+  }
+
+  void UnsafeShrinkRegistry() {
+    auto new_capacity = capacity / 2;
+    TRPC_CHECK(new_capacity);
+    auto new_stacks = new void*[new_capacity];
+    std::size_t copied = 0;
+
+    memset(new_stacks, 0, new_capacity * sizeof(void*));
+    for (std::size_t i = 0; i != capacity; ++i) {
+      if (stacks[i]) {
+        new_stacks[copied++] = stacks[i];
+      }
+    }
+
+    TRPC_CHECK_EQ(copied, used);
+    TRPC_CHECK_LE(copied, new_capacity);
+    capacity = new_capacity;
+    delete[] std::exchange(stacks, new_stacks);
+  }
+
+  void UnsafeResizeRegistry() {
+    if (capacity == 0) {  // We haven't been initialized yet.
+      capacity = 8;
+      stacks = new void*[capacity];
+      memset(stacks, 0, sizeof(void*) * capacity);
+    } else {
+      auto new_capacity = capacity * 2;
+      auto new_stacks = new void*[new_capacity];
+      memset(new_stacks, 0, new_capacity * sizeof(void*));
+      memcpy(new_stacks, stacks, capacity * sizeof(void*));
+      capacity = new_capacity;
+      delete[] std::exchange(stacks, new_stacks);
+    }
+  }
+
+ private:
+  std::mutex lock_;
+} stack_registry;  // Using global variable here. This makes looking up this
+                   // variable easy in GDB plugin.
+
+// End of source codes that are from flare.
 
 void SetFiberPoolCreateWay(bool use_mmap) {
   fiber_use_mmap = use_mmap;
 }
 
-const uint32_t kPageSize = getpagesize();
-
 void SetFiberStackSize(uint32_t stack_size) {
   bool ok = (stack_size >= kPageSize) && ((stack_size & (kPageSize - 1)) == 0);
   if (!ok) {
@@ -58,6 +174,10 @@ void SetFiberPoolNumByMmap(uint32_t fiber_pool_num_by_mmap) {
   max_fiber_num_by_mmap = fiber_pool_num_by_mmap;
 }
 
+void SetEnableGdbDebug(bool flag) {
+  enable_gdb_debug = flag;
+}
+
 // We always align stack top to 1M boundary. This helps our GDB plugin to find
 // fiber stacks.
 constexpr auto kStackTopAlignment = 1 * 1024 * 1024;
@@ -141,10 +261,23 @@ void* AlignedMmap() {
   }
   auto stack = reinterpret_cast<char*>(p) + GetBias();
   InitializeFiberStackMagic(stack + fiber_stack_size - kPageSize);
+
+  if (enable_gdb_debug) {
+    auto stack_bottom = stack + fiber_stack_size;
+    // Register the stack.
+    stack_registry.RegisterStack(stack_bottom);
+  }
+
   return reinterpret_cast<void*>(stack);
 }
 
 void AlignedMunmap(void* ptr) {
+  if (enable_gdb_debug) {
+    // Remove the stack from our registry.
+    auto stack_bottom = reinterpret_cast<char*>(ptr) + fiber_stack_size;
+    stack_registry.DeregisterStack(stack_bottom);
+  }
+
   TRPC_PCHECK(munmap(reinterpret_cast<char*>(ptr) - GetBias(), GetAllocationsize()) == 0);
 }
 
@@ -155,10 +288,23 @@ void* AlignedMalloc() {
   }
   auto stack = reinterpret_cast<char*>(p);
   InitializeFiberStackMagic(stack + fiber_stack_size - kPageSize);
+
+  if (enable_gdb_debug) {
+    auto stack_bottom = stack + fiber_stack_size;
+    // Register the stack.
+    stack_registry.RegisterStack(stack_bottom);
+  }
+
   return reinterpret_cast<void*>(stack);
 }
 
 void AlignedFree(void* aligned_mem) {
+  if (enable_gdb_debug) {
+    // Remove the stack from our registry.
+    auto stack_bottom = reinterpret_cast<char*>(aligned_mem) + fiber_stack_size;
+    stack_registry.DeregisterStack(stack_bottom);
+  }
+
   return free(aligned_mem);
 }
 

trpc/runtime/threadmodel/fiber/detail/stack_allocator_impl.h

@@ -36,6 +36,9 @@ void SetFiberStackEnableGuardPage(bool flag);
 /// @brief Set the number of pooled memory stacks allocated through mmap used by fibers
 void SetFiberPoolNumByMmap(uint32_t fiber_pool_num_by_mmap);
 
+/// @brief Enable debug fiber using gdb
+void SetEnableGdbDebug(bool flag);
+
 /// @brief Pre-allocate a certain number of fiber memory stacks
 /// @return The number of successfully warmed up fiber memory blocks
 int PrewarmFiberPool(uint32_t fiber_num);