Enhancement: Improve tracing performance (#989) (#990)

* Use :erlang.trace instead of :dbg.tracer

* Change :procs flag applying

* Update comments

* Refactor and add tests

* CR suggestions

Co-authored-by: Hubert Kasprzycki <67634024+hhubert6@users.noreply.github.com>

Author: GuzekAlan

lib/live_debugger/api/system/dbg.ex

@@ -1,13 +1,17 @@
 defmodule LiveDebugger.API.System.Dbg do
   @moduledoc """
-  API for interacting with the Erlang's `:dbg` tracing functionalities.
+  API for interacting with Erlang's tracing functionalities.
+
+  The implementation uses `:erlang.trace/3` and `:erlang.trace_pattern/3` BIFs
+  directly (instead of going through `:dbg`) to reduce per-message overhead in
+  the tracer.
 
   ## Usage
   1. If you want to trace callbacks you need to start tracer using `tracer/1` function.
   Do not start it multiple times, as it will return an error if the tracer is already running.
   2. Then using `process/1` you can enable tracing for all processes in the system.
   3. To add a trace pattern for a specific function or module, use `trace_pattern/2`.
-  If you want to trace more information you can pass `match_spec` created with `flags_to_match_spec/1` function.
+  If you want to trace more information you can pass `match_spec` created with `flag_to_match_spec/1` function.
   """
 
   @tracing_flags [:return_trace, :exception_trace]
@@ -24,35 +28,42 @@ defmodule LiveDebugger.API.System.Dbg do
   @callback stop() :: :ok
 
   @doc """
-  Starts tracer process and returns its PID.
-  When tracer is already started, it returns error.
-  It uses `:dbg.tracer/2` under the hood.
+  Starts a tracer process and returns its PID.
+  When a tracer is already started, returns an error.
+
+  The tracer is a plain Erlang process that loops on `receive` and calls
+  the supplied handler for every trace message produced by the BEAM tracing
+  BIFs. This mirrors what `:dbg.tracer(:process, handler)` did, but without
+  the `:dbg` server in the middle.
   """
   @spec tracer(handler_spec()) :: {:ok, pid()} | {:error, term()}
   def tracer(handler_spec), do: impl().tracer(handler_spec)
 
   @doc """
-  Enables tracing for all processes in the system.
+  Enables tracing for all processes in the system using `:erlang.trace/3`.
 
-  For list of supported flags, see `:dbg.p/2`.
+  Accepts the same short flag mnemonics as `:dbg.p/2` (e.g. `:c`, `:s`, `:p`)
+  for compatibility with the existing call sites; they are translated into
+  the BIF's full flag names internally. Long names (e.g. `:call`, `:timestamp`,
+  `:procs`) pass through unchanged.
   """
   @spec process(flags :: list()) :: {:ok, term()} | {:error, term()}
   def process(flags) when is_list(flags) do
     impl().process(flags)
   end
 
   @doc """
-  Enables tracing for a specific process.
+  Enables tracing for a specific process using `:erlang.trace/3`.
   """
   @spec process(pid(), flags :: list()) :: {:ok, term()} | {:error, term()}
   def process(pid, flags) when is_list(flags) and is_pid(pid) do
     impl().process(pid, flags)
   end
 
   @doc """
-  This is a wrapper for `:dbg.tp/2`.
   Adds a trace pattern for the specified module or MFA (Module, Function, Arity).
-  You can create proper `match_spec` by using `flags_to_match_spec/1` function.
+  This is a wrapper for `:erlang.trace_pattern/3` with `:global` scope.
+  You can create proper `match_spec` by using `flag_to_match_spec/1` function.
   """
   @spec trace_pattern(module() | mfa(), match_spec :: term()) ::
           {:ok, term()} | {:error, term()}
@@ -62,24 +73,23 @@ defmodule LiveDebugger.API.System.Dbg do
 
   @doc """
   Removes a trace pattern for the specified module or MFA (Module, Function, Arity).
-  This is a wrapper for `:dbg.ctp/1`.
+  This is a wrapper for `:erlang.trace_pattern/3` with `false` match spec.
   """
   @spec clear_trace_pattern(module() | mfa()) :: {:ok, term()} | {:error, term()}
   def clear_trace_pattern(module_or_mfa) do
     impl().clear_trace_pattern(module_or_mfa)
   end
 
   @doc """
-  Stops dbg process.
-  This is a wrapper for `:dbg.stop/0`
+  Stops the tracer process and clears all active trace flags and patterns.
   """
   @spec stop() :: :ok
   def stop() do
     impl().stop()
   end
 
   @doc """
-  Converts flag to `match_spec` format used by `tp/2` function.
+  Converts flag to `match_spec` format used by `trace_pattern/2`.
   Available flags are: #{Enum.map_join(@tracing_flags, ", ", &"`:#{&1}`")}
   """
   @spec flag_to_match_spec(flag :: atom()) :: term()
@@ -99,34 +109,126 @@ defmodule LiveDebugger.API.System.Dbg do
     @moduledoc false
     @behaviour LiveDebugger.API.System.Dbg
 
+    # Registered name of the tracer process. The `process/1,2` calls look it
+    # up here because `:erlang.trace/3` requires the tracer pid to be passed
+    # alongside the trace flags.
+    @tracer_name :live_debugger_tracer
+
+    # Translation table for the short flag mnemonics historically accepted by
+    # `:dbg.p/2`. Long names pass through unchanged via the default branch in
+    # `translate_flag/1`.
+    @flag_translation %{
+      s: :send,
+      r: :receive,
+      m: :messages,
+      c: :call,
+      p: :procs,
+      sos: :set_on_spawn,
+      sol: :set_on_link,
+      sofs: :set_on_first_spawn,
+      sofl: :set_on_first_link
+    }
+
     @impl true
-    def tracer(handler) do
-      :dbg.tracer(:process, handler)
+    def tracer({handler_fun, initial_data})
+        when is_function(handler_fun, 2) do
+      pid = spawn(fn -> loop(handler_fun, initial_data) end)
+
+      try do
+        Process.register(pid, @tracer_name)
+        {:ok, pid}
+      rescue
+        ArgumentError ->
+          Process.exit(pid, :kill)
+          {:error, :already_started}
+      end
     end
 
     @impl true
-    def process(flags) do
-      :dbg.p(:all, flags)
-    end
+    def process(flags), do: do_process(:all, flags)
 
     @impl true
-    def process(pid, flags) do
-      :dbg.p(pid, flags)
-    end
+    def process(pid, flags), do: do_process(pid, flags)
 
     @impl true
     def trace_pattern(pattern, match_spec) do
-      :dbg.tp(pattern, match_spec)
+      result = :erlang.trace_pattern(normalize_pattern(pattern), match_spec, [])
+      {:ok, result}
+    catch
+      kind, reason -> {:error, {kind, reason}}
     end
 
     @impl true
     def clear_trace_pattern(pattern) do
-      :dbg.ctp(pattern)
+      result = :erlang.trace_pattern(normalize_pattern(pattern), false, [])
+      {:ok, result}
+    catch
+      kind, reason -> {:error, {kind, reason}}
     end
 
     @impl true
     def stop() do
-      :dbg.stop()
+      safe(fn -> :erlang.trace(:all, false, [:all]) end)
+      safe(fn -> :erlang.trace_pattern({:_, :_, :_}, false, []) end)
+
+      case Process.whereis(@tracer_name) do
+        nil ->
+          :ok
+
+        pid ->
+          ref = Process.monitor(pid)
+          send(pid, :stop)
+
+          receive do
+            {:DOWN, ^ref, _, _, _} -> :ok
+          after
+            1_000 ->
+              Process.exit(pid, :kill)
+              :ok
+          end
+      end
+    end
+
+    # Tracer process loop. Receives raw trace tuples from the BEAM and
+    # dispatches them to the user-provided handler, threading state through
+    # each call. A `:stop` message terminates the loop with reason `:done`.
+    defp loop(handler_fun, data) do
+      receive do
+        :stop ->
+          exit(:done)
+
+        msg ->
+          new_data = handler_fun.(msg, data)
+          loop(handler_fun, new_data)
+      end
+    end
+
+    defp do_process(target, flags) do
+      case Process.whereis(@tracer_name) do
+        nil ->
+          {:error, :tracer_not_started}
+
+        tracer ->
+          result =
+            :erlang.trace(target, true, [{:tracer, tracer} | translate_flags(flags)])
+
+          {:ok, result}
+      end
+    catch
+      kind, reason -> {:error, {kind, reason}}
+    end
+
+    defp normalize_pattern({_mod, _fun, _arity} = mfa), do: mfa
+    defp normalize_pattern(module) when is_atom(module), do: {module, :_, :_}
+
+    defp translate_flags(flags), do: Enum.map(flags, &translate_flag/1)
+    defp translate_flag(flag), do: Map.get(@flag_translation, flag, flag)
+
+    defp safe(fun) do
+      fun.()
+      :ok
+    catch
+      _, _ -> :ok
     end
   end
 end

lib/live_debugger/services/callback_tracer/README.md

@@ -1,7 +1,7 @@
 ## CallbackTracer
 
-This service is responsible for setting up `:dbg.tracer`, monitoring and managing it, parsing raw traces and finally sending them to the bus.
+This service is responsible for setting up the tracer process, monitoring and managing it, parsing raw traces and finally sending them to the bus.
 
 ### Overview
 
-The CallbackTracer initializes `:dbg.tracer` to collect callback-related data. It continuously monitors the tracer for errors and exceptions. Trace data sent from the tracer is received by a GenServer within the service, which then analyzes and persists this data before broadcasting it to the event bus. The service also adapts tracer configurations in response to system changes like module recompilations to maintain accurate and relevant data collection.
+The CallbackTracer starts a dedicated tracer process (via `:erlang.trace/3` / `:erlang.trace_pattern/3`) to collect callback-related data. It continuously monitors the tracer for errors and exceptions. Trace data sent from the tracer is received by a GenServer within the service, which then analyzes and persists this data before broadcasting it to the event bus. The service also adapts tracer configurations in response to system changes like module recompilations to maintain accurate and relevant data collection.

lib/live_debugger/services/callback_tracer/actions/tracing.ex

@@ -24,35 +24,32 @@ defmodule LiveDebugger.Services.CallbackTracer.Actions.Tracing do
   def setup_tracing_with_monitoring!(state) do
     last_id = TraceQueries.get_last_trace_id()
 
-    if Map.get(state, :dbg_pid) do
-      Dbg.stop()
-    end
-
-    case Dbg.tracer({&Tracer.handle_trace/2, {:init, last_id - 1}}) do
-      {:ok, pid} ->
-        Process.monitor(pid)
+    Dbg.stop()
 
-        Dbg.process([:c, :timestamp, :procs])
+    with {:ok, pid} <- Dbg.tracer({&Tracer.handle_trace/2, {:init, last_id - 1}}),
+         {:ok, _} <- Dbg.process([:c, :timestamp]) do
+      Process.monitor(pid)
 
-        # Fetch all live modules once with paths for both operations
-        live_modules_with_paths = CallbackQueries.all_live_modules_with_paths()
+      # Fetch all live modules once with paths for both operations
+      live_modules_with_paths = CallbackQueries.all_live_modules_with_paths()
 
-        # Extract just the module names for callback queries
-        module_names = Enum.map(live_modules_with_paths, fn {module, _path} -> module end)
+      # Extract just the module names for callback queries
+      module_names = Enum.map(live_modules_with_paths, fn {module, _path} -> module end)
 
-        # Apply trace patterns using the fetched modules
-        apply_trace_patterns(module_names)
+      # Apply trace patterns using the fetched modules
+      apply_trace_patterns(module_names)
 
-        # Monitor recompilation using the paths
-        start_file_monitoring(live_modules_with_paths)
+      # Monitor recompilation using the paths
+      start_file_monitoring(live_modules_with_paths)
 
-        # Broadcast information
-        Bus.broadcast_event!(%DbgStarted{})
-
-        %{state | dbg_pid: pid}
+      # Broadcast information
+      Bus.broadcast_event!(%DbgStarted{})
 
+      %{state | dbg_pid: pid}
+    else
       {:error, error} ->
-        raise "Couldn't start tracer: #{inspect(error)}"
+        Logger.error("Couldn't start tracer: #{inspect(error)}")
+        state
     end
   end
 
@@ -71,9 +68,13 @@ defmodule LiveDebugger.Services.CallbackTracer.Actions.Tracing do
 
   @spec start_outgoing_messages_tracing(pid()) :: :ok
   def start_outgoing_messages_tracing(pid) do
-    Dbg.process(pid, [:s])
+    case Dbg.process(pid, [:s, :procs]) do
+      {:ok, _} ->
+        :ok
 
-    :ok
+      {:error, error} ->
+        Logger.error("Couldn't enable send tracing for #{inspect(pid)}: #{inspect(error)}")
+    end
   end
 
   defp refresh_tracing_for_module(module) do

lib/live_debugger/services/callback_tracer/gen_servers/trace_handler.ex

@@ -46,7 +46,7 @@ defmodule LiveDebugger.Services.CallbackTracer.GenServers.TraceHandler do
   end
 
   @doc """
-  Handles trace from `:dbg.tracer` process.
+  Handles trace from the tracer process started via `LiveDebugger.API.System.Dbg`.
   """
   @spec handle_trace(trace :: term(), n :: integer()) :: :ok
   def handle_trace(trace, n) do
@@ -146,7 +146,7 @@ defmodule LiveDebugger.Services.CallbackTracer.GenServers.TraceHandler do
   #########################################################
   # Handling diffs tracing
   #
-  # Use :dbg.p(channel_pid, [:s]) to activate
+  # Activated via `Dbg.process(channel_pid, [:s])` from `TracingActions`.
   #########################################################
 
   def handle_cast({:new_trace, {_, pid, :send, {:socket_push, :text, iodata}, _, ts}, n}, state) do

lib/live_debugger/services/callback_tracer/gen_servers/tracing_manager.ex

@@ -81,8 +81,7 @@ defmodule LiveDebugger.Services.CallbackTracer.GenServers.TracingManager do
   end
 
   # Handling tracer process stop or crash.
-  # All exit messages are trapped and sent with `:done` reason.
-  def handle_info({:DOWN, _, _, pid, :done}, %{dbg_pid: pid} = state) do
+  def handle_info({:DOWN, _, _, pid, _reason}, %{dbg_pid: pid} = state) do
     Bus.broadcast_event!(%DbgKilled{})
 
     {:noreply, %{state | dbg_pid: nil}}

lib/live_debugger/services/callback_tracer/process/tracer.ex

@@ -1,6 +1,7 @@
 defmodule LiveDebugger.Services.CallbackTracer.Process.Tracer do
   @moduledoc """
-  This module defines a function that is used in the `:dbg.tracer` process.
+  This module defines a function that is used as the handler in the tracer process
+  started by `LiveDebugger.API.System.Dbg.tracer/1`.
   """
 
   alias LiveDebugger.Services.CallbackTracer.GenServers.TraceHandler
@@ -10,7 +11,8 @@ defmodule LiveDebugger.Services.CallbackTracer.Process.Tracer do
 
   @spec handle_trace(trace :: term(), state :: integer() | {:init, integer()}) :: integer()
   def handle_trace(trace, {:init, n}) do
-    # Maximum heap size is set inside `:dbg.tracer` process to prevent it from consuming too much memory.
+    # Set on first invocation so the limit applies to the tracer process itself
+    # (which is the one receiving every BEAM trace message).
     Memory.set_max_heap_size(@max_heap_size)
 
     do_handle_trace(trace, n)

lib/live_debugger/structs/trace.ex

@@ -6,7 +6,7 @@ defmodule LiveDebugger.Structs.Trace do
   - DiffTrace - represents a diff trace (HTML diff generated by LiveView/LiveComponent sent to the client).
 
   Each trace has:
-  - id - unique id of the trace which is generated by :dbg tracer.
+  - id - unique id of the trace, generated by the tracer process.
   - pid - pid of the LiveView that generated the trace.
   - timestamp - timestamp of the trace.
   """