fix(elixir): deterministic KRaft 3-node propose + guard telemetry entity count

The "3-node cluster persists state across all nodes" recovery test read
diagnostics once, picked a node reporting :leader, then proposed to it. On a
freshly-started cluster a re-election can step that node down inside the race
window, so propose returned {:error, {:not_leader, _}} and the {:ok, _} match
crashed (kraft_recovery_test.exs:249, the lone failure on run #90).

Replace snapshot-then-propose with an await-leader barrier: poll for a node
that is leader, propose to it, follow any {:not_leader, _} redirect and retry
across re-elections until the cluster commits or a deadline elapses. A
deterministic harness rather than a fixed sleep, per #110.

Also guard VeriSim.Telemetry.measure_entity_count/0 against the entity
registry not being started (test env): catch the "unknown registry"
ArgumentError and report 0, so telemetry_poller no longer logs a crashed
measurement on every tick.

Refs #110.

https://claude.ai/code/session_01W9Voe3JceP66Bna9FT4jME

Author: claude

elixir-orchestration/lib/verisim/telemetry.ex

@@ -197,16 +197,25 @@ defmodule VeriSim.Telemetry do
 
   @doc false
   def measure_entity_count do
-    # Count entity servers
+    # Count entity servers. The registry may not be running (e.g. in the test
+    # env, where the full supervision tree isn't started), so guard against the
+    # "unknown registry" ArgumentError rather than letting telemetry_poller log
+    # a crashed measurement on every tick.
     count =
-      case Registry.count(VeriSim.EntityRegistry) do
+      case registered_entity_count() do
         n when is_integer(n) -> n
         _ -> 0
       end
 
     :telemetry.execute([:verisim, :entities], %{count: count}, %{})
   end
 
+  defp registered_entity_count do
+    Registry.count(VeriSim.EntityRegistry)
+  rescue
+    ArgumentError -> 0
+  end
+
   @doc false
   def measure_drift_status do
     # Get drift status from monitor

elixir-orchestration/test/verisim/consensus/kraft_recovery_test.exs

@@ -37,6 +37,55 @@ defmodule VeriSim.Consensus.KRaftRecoveryTest do
     :exit, _ -> :ok
   end
 
+  # Propose `command` through whichever node is currently the leader, retrying
+  # across re-elections and following {:not_leader, _} redirects, until the
+  # cluster accepts it (-> {:ok, index}) or `timeout_ms` elapses. Deterministic
+  # alternative to "read diagnostics once, propose once", which races with a
+  # leader step-down on a freshly-started cluster.
+  defp propose_when_leader(ids, command, timeout_ms \\ 5_000) do
+    deadline = System.monotonic_time(:millisecond) + timeout_ms
+    do_propose_when_leader(ids, command, deadline)
+  end
+
+  defp do_propose_when_leader(ids, command, deadline) do
+    leader =
+      Enum.find_value(ids, fn id ->
+        case safe_diagnostics(id) do
+          %{role: :leader} -> id
+          _ -> nil
+        end
+      end)
+
+    result = if leader, do: safe_propose(leader, command), else: {:error, :no_leader}
+
+    case result do
+      {:ok, _} = ok ->
+        ok
+
+      _other ->
+        if System.monotonic_time(:millisecond) < deadline do
+          Process.sleep(50)
+          do_propose_when_leader(ids, command, deadline)
+        else
+          result
+        end
+    end
+  end
+
+  # GenServer.call wrappers that tolerate a node being mid-transition: a call can
+  # time out or the process can be briefly unavailable during an election.
+  defp safe_diagnostics(id) do
+    KRaftNode.diagnostics(id)
+  catch
+    :exit, _ -> %{role: :unknown}
+  end
+
+  defp safe_propose(id, command) do
+    KRaftNode.propose(id, command)
+  catch
+    :exit, _ -> {:error, :unavailable}
+  end
+
   # ===========================================================================
   # WAL persistence during normal operation
   # ===========================================================================
@@ -236,22 +285,19 @@ defmodule VeriSim.Consensus.KRaftRecoveryTest do
           start_node(id, peers: peers, wal_path: wal_path)
         end)
 
-      # Wait for election
-      Process.sleep(1_500)
-
-      # Find the leader
-      {leader_id, _} =
-        ids
-        |> Enum.map(fn id -> {id, KRaftNode.diagnostics(id)} end)
-        |> Enum.find(fn {_id, diag} -> diag.role == :leader end)
-
-      # Propose a command through the leader
-      {:ok, _} =
-        KRaftNode.propose(
-          leader_id,
-          {:register_store, "cluster-store", "http://cluster:8080", ["graph"]}
-        )
-
+      # Propose through whichever node is currently the leader, retrying across
+      # re-elections. A freshly-started 3-node cluster can re-elect during the
+      # first election cycles, so reading diagnostics once and proposing to that
+      # node races with a step-down (-> {:error, {:not_leader, _}}). Await a node
+      # that is leader *and* accepts the proposal, following any not_leader
+      # redirect — a deterministic barrier rather than a fixed sleep.
+      assert {:ok, _} =
+               propose_when_leader(
+                 ids,
+                 {:register_store, "cluster-store", "http://cluster:8080", ["graph"]}
+               )
+
+      # Let the commit replicate to the followers so every WAL records a term.
       Process.sleep(500)
 
       # All nodes should have WAL data