chore: Distributed MapCache (blockscout#14411)

Co-authored-by: Victor Baranov <baranov.viktor.27@gmail.com>

Author: Qwerty5Uiop

apps/explorer/lib/explorer/chain/map_cache.ex

@@ -45,6 +45,12 @@ defmodule Explorer.Chain.MapCache do
   - `{:return, value}` that will cause the value to be returned but not stored
   This allows to define of a default value or perform some actions.
   By default it will simply `{:return, nil}`
+
+  ## Distributed writes
+
+  In split API/indexer deployments, `set/2` and `update/2` use `do_raw/2`: the write runs on
+  the local `ConCache` first, then indexer nodes multicast the same operation to other cluster
+  nodes via `:erpc` (`propagate: false` on receivers). Reads are not distributed.
   """
 
   @type key :: atom()
@@ -114,6 +120,7 @@ defmodule Explorer.Chain.MapCache do
 
     # credo:disable-for-next-line Credo.Check.Refactor.LongQuoteBlocks
     quote do
+      require Logger
       alias Explorer.Chain.MapCache
 
       @behaviour MapCache
@@ -156,7 +163,7 @@ defmodule Explorer.Chain.MapCache do
 
       @impl MapCache
       def set(key, value) do
-        ConCache.put(cache_name(), key, value)
+        do_raw(fn -> ConCache.put(cache_name(), key, value) end)
       end
 
       @impl MapCache
@@ -169,7 +176,31 @@ defmodule Explorer.Chain.MapCache do
 
       @impl MapCache
       def update(key, value) do
-        ConCache.update(cache_name(), key, fn old_val -> handle_update(key, old_val, value) end)
+        do_raw(fn -> ConCache.update(cache_name(), key, fn old_val -> handle_update(key, old_val, value) end) end)
+      end
+
+      @doc """
+      Applies a ConCache write locally, then propagates it from indexer nodes.
+
+      With the default `propagate: true`, the given zero-arity function runs on this node first.
+      When `Explorer.mode/0` is `:indexer`, the same function is multicast to `Node.list/0` with
+      `propagate: false` so API nodes apply the write without re-propagating.
+      """
+      def do_raw(function, propagate \\ true) do
+        function.()
+
+        case Explorer.mode() do
+          :indexer ->
+            if propagate do
+              Node.list() |> :erpc.multicast(__MODULE__, :do_raw, [function, false])
+            else
+              Logger.error("[#{__MODULE__}] Indexer got unexpected propagation call to do_raw/2")
+              :ok
+            end
+
+          _ ->
+            :ok
+        end
       end
 
       ### Autogenerated named functions

apps/explorer/lib/explorer/chain/ordered_cache.ex

@@ -36,6 +36,12 @@ defmodule Explorer.Chain.OrderedCache do
   and `c:element_to_id/1` callbacks.
   For typechecking purposes it's also recommended to override the `t:element/0`
   and `t:id/0` type definitions.
+
+  ## Distributed writes
+
+  In split API/indexer deployments, `update/1` uses `do_raw_update/2`: the ids list and
+  elements are written to the local `ConCache` first, then indexer nodes multicast the prepared
+  update to other cluster nodes via `:erpc` (`propagate: false` on receivers).
   """
 
   @type element :: struct()
@@ -290,18 +296,24 @@ defmodule Explorer.Chain.OrderedCache do
 
       def update(element), do: update([element])
 
+      @doc """
+      Merges prepared elements into the local ordered cache, then propagates from indexer nodes.
+
+      Always updates the local ids list and element entries first. When `Explorer.mode/0` is
+      `:indexer` and `propagate` is `true`, multicasts the same prepared elements to `Node.list/0`
+      with `propagate: false` so API nodes apply the write without re-propagating.
+      """
       def do_raw_update(prepared_elements, propagate) do
-        case Explorer.mode() do
-          mode when mode in [:all, :api] ->
-            ConCache.update(cache_name(), ids_list_key(), fn ids ->
-              updated_list =
-                prepared_elements
-                |> merge_and_update(ids || [], max_size())
+        ConCache.update(cache_name(), ids_list_key(), fn ids ->
+          updated_list =
+            prepared_elements
+            |> merge_and_update(ids || [], max_size())
 
-              # ids_list is set to never expire
-              {:ok, %ConCache.Item{value: updated_list, ttl: :infinity}}
-            end)
+          # ids_list is set to never expire
+          {:ok, %ConCache.Item{value: updated_list, ttl: :infinity}}
+        end)
 
+        case Explorer.mode() do
           :indexer ->
             if propagate do
               Node.list() |> :erpc.multicast(__MODULE__, :do_raw_update, [prepared_elements, false])

apps/explorer/test/explorer/chain/cache/transactions_test.exs

@@ -85,6 +85,33 @@ defmodule Explorer.Chain.Cache.TransactionsTest do
     end
   end
 
+  describe "indexer mode" do
+    setup do
+      old_mode = Application.get_env(:explorer, :mode)
+      Application.put_env(:explorer, :mode, :indexer)
+
+      Supervisor.terminate_child(Explorer.Supervisor, Transactions.child_id())
+      Supervisor.restart_child(Explorer.Supervisor, Transactions.child_id())
+
+      on_exit(fn -> Application.put_env(:explorer, :mode, old_mode) end)
+
+      :ok
+    end
+
+    test "update/1 writes to local ConCache without connected API nodes" do
+      assert Explorer.mode() == :indexer
+      # Single-node test env: propagation multicasts to an empty list, so only the
+      # local-first write in do_raw_update/2 populates the cache.
+      assert Node.list() == []
+
+      transaction = insert(:transaction) |> preload_all()
+
+      Transactions.update(transaction)
+
+      assert Transactions.take(1) == [transaction]
+    end
+  end
+
   defp preload_all(transactions) when is_list(transactions) do
     Enum.map(transactions, &preload_all(&1))
   end

cspell.json

@@ -450,6 +450,7 @@
         "mtransfer",
         "mult",
         "multicall",
+        "multicasts",
         "multichain",
         "multiprotocol",
         "multis",