diff --git a/.github/workflows/elixir-ci.yml b/.github/workflows/elixir-ci.yml index a0ed9f2..7df8a89 100644 --- a/.github/workflows/elixir-ci.yml +++ b/.github/workflows/elixir-ci.yml @@ -57,7 +57,10 @@ jobs: - run: mix test coverage: - name: ExCoveralls (≥60%) + # Unit-test coverage only: the federation adapters are exercised by the + # :integration suite (live databases; excluded in CI) and are skipped in + # coveralls.json, so they don't dilute the unit-coverage denominator. + name: ExCoveralls (unit, gate in coveralls.json) runs-on: ubuntu-latest defaults: run: diff --git a/elixir-orchestration/coveralls.json b/elixir-orchestration/coveralls.json index 2eb5a22..dffdd4a 100644 --- a/elixir-orchestration/coveralls.json +++ b/elixir-orchestration/coveralls.json @@ -11,6 +11,7 @@ "test/", "deps/", "_build/", - "lib/mix/tasks/" + "lib/mix/tasks/", + "lib/verisim/federation/adapters/" ] } diff --git a/elixir-orchestration/lib/verisim/consensus/kraft_node.ex b/elixir-orchestration/lib/verisim/consensus/kraft_node.ex index 20648b5..7712676 100644 --- a/elixir-orchestration/lib/verisim/consensus/kraft_node.ex +++ b/elixir-orchestration/lib/verisim/consensus/kraft_node.ex @@ -326,19 +326,25 @@ defmodule VeriSim.Consensus.KRaftNode do last_log_term: last_log_term } - for peer <- state.peers do + peers = effective_peers(state) + + for peer <- peers do Task.start(fn -> try do response = GenServer.call(via(peer), {:request_vote, request}, 1_000) send(self_pid(state.node_id), {:vote_response, peer, response}) rescue _ -> :timeout + catch + # Unreachable peer (not registered / stopped): a failed RPC, not a + # crash — the election timer drives the retry. + :exit, _ -> :unreachable end end) end # Check if we already have quorum (e.g., single-node cluster with 0 peers) - quorum = div(length(state.peers) + 1, 2) + 1 + quorum = div(length(peers) + 1, 2) + 1 if MapSet.size(state.votes_received) >= quorum do become_leader(state) @@ -387,7 +393,7 @@ defmodule VeriSim.Consensus.KRaftNode do state.role == :candidate and response.vote_granted -> votes = MapSet.put(state.votes_received, from_peer) - quorum = div(length(state.peers) + 1, 2) + 1 + quorum = div(length(effective_peers(state)) + 1, 2) + 1 if MapSet.size(votes) >= quorum do become_leader(%{state | votes_received: votes}) @@ -450,7 +456,7 @@ defmodule VeriSim.Consensus.KRaftNode do # --------------------------------------------------------------------------- defp send_append_entries(state) do - for peer <- state.peers do + for peer <- effective_peers(state) do next_idx = Map.get(state.next_index, peer, 1) prev_log_index = next_idx - 1 @@ -483,6 +489,10 @@ defmodule VeriSim.Consensus.KRaftNode do send(self_pid(node_id), {:append_entries_response, peer, response}) rescue _ -> :timeout + catch + # Unreachable peer (not registered / stopped): a failed RPC, not a + # crash — the next heartbeat drives the retry. + :exit, _ -> :unreachable end end) end @@ -577,12 +587,16 @@ defmodule VeriSim.Consensus.KRaftNode do # --------------------------------------------------------------------------- defp maybe_advance_commit(state) do - # Find highest N where majority of matchIndex[i] >= N + # Find highest N where a majority of matchIndex[i] >= N. Membership is + # the latest configuration in the log (effective_peers/1), and only + # members of that configuration count toward the quorum. + peers = effective_peers(state) + all_match = - (Map.values(state.match_index) ++ [length(state.log)]) + (Enum.map(peers, &Map.get(state.match_index, &1, 0)) ++ [length(state.log)]) |> Enum.sort(:desc) - quorum_idx = div(length(state.peers) + 1, 2) + quorum_idx = div(length(peers) + 1, 2) new_commit = Enum.at(all_match, quorum_idx, state.commit_index) # Only commit entries from current term (Raft safety property) @@ -680,6 +694,19 @@ defmodule VeriSim.Consensus.KRaftNode do defp apply_membership_change(peers, _self_id, _command), do: peers + # Raft single-server membership rule (Ongaro, "Consensus" §4.1): a server + # uses the LATEST configuration present in its log — committed or not — + # for quorum counting and replication targets. `state.peers` is the + # configuration as of `last_applied`; replay any unapplied config deltas + # in the log suffix on top of it. + defp effective_peers(state) do + state.log + |> Enum.drop(state.last_applied) + |> Enum.reduce(state.peers, fn entry, peers -> + apply_membership_change(peers, state.node_id, entry.command) + end) + end + defp maybe_trigger_snapshot(state) do if state.wal_path != nil and state.last_applied > 0 and rem(state.last_applied, @snapshot_interval) == 0 do diff --git a/elixir-orchestration/test/test_helper.exs b/elixir-orchestration/test/test_helper.exs index bfa2917..0fa3406 100644 --- a/elixir-orchestration/test/test_helper.exs +++ b/elixir-orchestration/test/test_helper.exs @@ -1,3 +1,6 @@ # SPDX-License-Identifier: MPL-2.0 -ExUnit.start() +# Integration tests (federation adapters and anything else needing live +# external services) are excluded by default so plain `mix test` is +# self-contained. Run them with: mix test --include integration +ExUnit.start(exclude: [:integration]) diff --git a/elixir-orchestration/test/verisim/consensus/kraft_node_test.exs b/elixir-orchestration/test/verisim/consensus/kraft_node_test.exs index 2d7276c..8eab0e5 100644 --- a/elixir-orchestration/test/verisim/consensus/kraft_node_test.exs +++ b/elixir-orchestration/test/verisim/consensus/kraft_node_test.exs @@ -36,6 +36,20 @@ defmodule VeriSim.Consensus.KRaftNodeTest do :exit, _ -> :ok end + defp wait_for_leader(node_id, attempts \\ 50) do + case KRaftNode.diagnostics(node_id) do + %{role: :leader} -> + :ok + + _ when attempts > 0 -> + Process.sleep(20) + wait_for_leader(node_id, attempts - 1) + + _ -> + flunk("node #{node_id} did not become leader") + end + end + describe "single-node leader election" do test "node with 0 peers becomes leader within 500ms" do node_id = "solo-#{System.unique_integer([:positive])}" @@ -155,40 +169,55 @@ defmodule VeriSim.Consensus.KRaftNodeTest do describe "dynamic membership — add_server" do test "leader accepts add_server and new peer appears in registry members" do - node_id = "add-#{System.unique_integer([:positive])}" + suffix = System.unique_integer([:positive]) + node_id = "add-#{suffix}" + peer_id = "add-peer-#{suffix}" + pid = start_kraft(node_id) + wait_for_leader(node_id) - Process.sleep(500) + # The peer must be a real, responding process: under the + # latest-config-in-log rule the add entry itself commits against the + # NEW configuration {leader, peer}, so the peer has to acknowledge + # replication. (You cannot add a server that never responds.) + peer_pid = start_kraft(peer_id, [node_id]) - assert {:ok, _index} = KRaftNode.add_server(node_id, "new-peer-1", []) + assert {:ok, _index} = KRaftNode.add_server(node_id, peer_id, [node_id]) Process.sleep(100) registry = KRaftNode.registry(node_id) members = get_in(registry, [:config, :members]) || [] - assert "new-peer-1" in members + assert peer_id in members + stop_kraft(peer_pid) stop_kraft(pid) end end describe "dynamic membership — remove_server" do test "leader accepts remove_server and peer is removed from registry members" do - node_id = "rm-#{System.unique_integer([:positive])}" - pid = start_kraft(node_id) + suffix = System.unique_integer([:positive]) + node_id = "rm-#{suffix}" + peer_id = "rm-peer-#{suffix}" - Process.sleep(500) + pid = start_kraft(node_id) + wait_for_leader(node_id) + peer_pid = start_kraft(peer_id, [node_id]) - # Add then remove - {:ok, _} = KRaftNode.add_server(node_id, "ephemeral-peer", []) + # Add a real peer, then remove it. The removal commits against the + # post-removal configuration {leader} (latest-config-in-log rule), so + # it cannot deadlock on the departing peer's acknowledgement. + {:ok, _} = KRaftNode.add_server(node_id, peer_id, [node_id]) Process.sleep(100) - {:ok, _} = KRaftNode.remove_server(node_id, "ephemeral-peer") + {:ok, _} = KRaftNode.remove_server(node_id, peer_id) Process.sleep(100) registry = KRaftNode.registry(node_id) members = get_in(registry, [:config, :members]) || [] - refute "ephemeral-peer" in members + refute peer_id in members + stop_kraft(peer_pid) stop_kraft(pid) end end diff --git a/elixir-orchestration/test/verisim/consensus/kraft_supervisor_test.exs b/elixir-orchestration/test/verisim/consensus/kraft_supervisor_test.exs index 62a8a70..b9f1041 100644 --- a/elixir-orchestration/test/verisim/consensus/kraft_supervisor_test.exs +++ b/elixir-orchestration/test/verisim/consensus/kraft_supervisor_test.exs @@ -39,8 +39,8 @@ defmodule VeriSim.Consensus.KRaftSupervisorTest do {:ok, {_flags, children}} = KRaftSupervisor.init(nodes: []) [registry_spec] = children - # Children come back as %{id: Registry, start: {Registry, :start_link, [...]}} - assert registry_spec.id == Registry + # Registry.child_spec/1 uses the :name option as the child id. + assert registry_spec.id == VeriSim.Consensus.Registry end test "one configured node yields Registry + one KRaftNode spec" do diff --git a/elixir-orchestration/test/verisim/query/vql_e2e_test.exs b/elixir-orchestration/test/verisim/query/vql_e2e_test.exs index f08eb44..9c2234a 100644 --- a/elixir-orchestration/test/verisim/query/vql_e2e_test.exs +++ b/elixir-orchestration/test/verisim/query/vql_e2e_test.exs @@ -28,7 +28,7 @@ defmodule VeriSim.Query.VQLE2ETest do use ExUnit.Case, async: false - alias VeriSim.Query.{VQLTypeChecker, VQLProofCertificate} + alias VeriSim.Query.{VQLBridge, VQLTypeChecker, VQLProofCertificate} alias VeriSim.Test.VQLTestHelpers, as: H setup_all do