feat(reaper): treat live mutable layers as live when reaping orphans

The reaper's liveness set was VMSupervisor children union cluster Routing,
which omits a vm whose mutable layer is alive but whose VM is mid-boot (not yet
routed) or in its post-stop idle grace. Union Img.Mutable.active_vm_ids/0 so the
reaper never removes a hyper-rw volume that a live owner still holds.

Author: markovejnovic

lib/hyper/node/reaper.ex

@@ -81,10 +81,19 @@ defmodule Hyper.Node.Reaper do
   end
 
   # Over-counting "live" only defers a reap (safe); under-counting destroys a live
-  # VM (catastrophic). So union two independent liveness sources: the local VM
-  # supervisor's children and the cluster routing table's view of this node.
+  # VM (catastrophic). Union three independent liveness sources: the local VM
+  # supervisor's children, the cluster routing table's view of this node, and the
+  # live mutable layers (which own the hyper-rw volumes and exist during the
+  # mid-boot and idle-grace windows when a vm is in neither of the other two).
   @spec gather_live() :: MapSet.t(String.t())
-  defp gather_live, do: MapSet.union(local_live(), routed_live())
+  defp gather_live do
+    local_live()
+    |> MapSet.union(routed_live())
+    |> MapSet.union(mutable_live())
+  end
+
+  @spec mutable_live() :: MapSet.t(String.t())
+  defp mutable_live, do: MapSet.new(Img.Mutable.active_vm_ids())
 
   @spec local_live() :: MapSet.t(String.t())
   defp local_live do

test/hyper/node/reaper/liveness_test.exs

@@ -0,0 +1,40 @@
+defmodule Hyper.Node.Reaper.LivenessTest do
+  use ExUnit.Case, async: false
+
+  alias Hyper.Node.Img
+  alias Hyper.Node.Img.Mutable
+  alias Hyper.Node.Reaper.Plan
+
+  setup do
+    name = Img.mutable_registry()
+
+    unless Process.whereis(name) do
+      start_supervised!({Registry, keys: :unique, name: name})
+    end
+
+    :ok
+  end
+
+  defp register_live(vm_id) do
+    test = self()
+
+    {:ok, _pid} =
+      Task.start_link(fn ->
+        {:ok, _} = Registry.register(Img.mutable_registry(), vm_id, nil)
+        send(test, {:registered, vm_id})
+        Process.sleep(:infinity)
+      end)
+
+    assert_receive {:registered, ^vm_id}
+  end
+
+  test "a vm with a live mutable layer is never an orphan, even with a leftover rw device" do
+    register_live("vm-live")
+
+    live = MapSet.new(Mutable.active_vm_ids())
+
+    # The reaper would see hyper-rw-vm-live in `dmsetup ls` (rw candidate) and no
+    # cgroup leaf, yet the live mutable owner must protect it from reaping.
+    assert Plan.orphans(live, [], ["vm-live"]) == MapSet.new([])
+  end
+end