Merge pull request #1099 from rocket-pool/monitor-ongoing-exits

Monitor ongoing exits

Author: 0xfornax

rocketpool-cli/megapool/commands.go

@@ -277,6 +277,10 @@ func RegisterCommands(app *cli.Command, name string, aliases []string) {
 						Name:  "yes",
 						Usage: "Automatically confirm the action",
 					},
+					&cli.BoolFlag{
+						Name:  "fetch-estimate",
+						Usage: "Fetch an estimate of the beacon chain exit queue time",
+					},
 					&cli.Uint64Flag{
 						Name:  "validator-id",
 						Usage: "The validator id to exit",
@@ -293,7 +297,7 @@ func RegisterCommands(app *cli.Command, name string, aliases []string) {
 					if !c.IsSet("validator-id") {
 						var err error
 						var found bool
-						validatorId, found, err = getExitableValidator()
+						validatorId, found, err = getExitableValidator(c.Bool("fetch-estimate"))
 						if err != nil {
 							return err
 						}

rocketpool-cli/megapool/exit-validator.go

@@ -3,7 +3,11 @@ package megapool
 import (
 	"fmt"
 	"sort"
+	"strconv"
+	"strings"
+	"time"
 
+	"github.com/rocket-pool/smartnode/shared/services/beacon"
 	"github.com/rocket-pool/smartnode/shared/services/rocketpool"
 	"github.com/rocket-pool/smartnode/shared/types/api"
 	"github.com/rocket-pool/smartnode/shared/utils/cli/color"
@@ -16,30 +20,142 @@ func (a ByIndex) Len() int           { return len(a) }
 func (a ByIndex) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a ByIndex) Less(i, j int) bool { return a[i].ValidatorIndex < a[j].ValidatorIndex }
 
-func getExitableValidator() (uint64, bool, error) {
+func formatGwei(gwei uint64) string {
+	eth := float64(gwei) / 1e9
+	if eth == float64(uint64(eth)) {
+		return fmt.Sprintf("%d", uint64(eth))
+	}
+	return fmt.Sprintf("%.2f", eth)
+}
+
+// formatDaysHours formats a duration as a "Xd Yh" string. Sub-hour values are
+// rendered in minutes; sub-minute values fall back to "<1m".
+func formatDaysHours(d time.Duration) string {
+	totalSeconds := int64(d.Seconds())
+	if totalSeconds < 60 {
+		return "<1m"
+	}
+	const secondsPerHour = 3600
+	const secondsPerDay = 24 * secondsPerHour
+	days := totalSeconds / secondsPerDay
+	hours := (totalSeconds % secondsPerDay) / secondsPerHour
+	if days > 0 {
+		return fmt.Sprintf("%dd %dh", days, hours)
+	}
+	if hours > 0 {
+		return fmt.Sprintf("%dh", hours)
+	}
+	minutes := totalSeconds / 60
+	return fmt.Sprintf("%dm", minutes)
+}
+
+// sortExitingValidatorsBySweep orders validators in withdrawal-sweep order relative to a given last validator index
+func sortExitingValidatorsBySweep(validators []api.MegapoolValidatorDetails, lastWithdrawnIndex uint64, hasLastWithdrawnIndex bool) {
+	if !hasLastWithdrawnIndex {
+		sort.Sort(ByIndex(validators))
+		return
+	}
+	sort.SliceStable(validators, func(i, j int) bool {
+		ai := validators[i].ValidatorIndex
+		aj := validators[j].ValidatorIndex
+		iAfter := ai > lastWithdrawnIndex
+		jAfter := aj > lastWithdrawnIndex
+		if iAfter != jAfter {
+			return iAfter
+		}
+		return ai < aj
+	})
+}
+
+func getExitableValidator(fetchExitQueueEstimate bool) (uint64, bool, error) {
 	// Get RP client
 	rp, err := rocketpool.NewClient().WithReady()
 	if err != nil {
 		return 0, false, err
 	}
 	defer rp.Close()
 
+	// Get the latest block and identify the withdrawals present in it
+	var lastWithdrawnIndex uint64
+	var hasLastWithdrawnIndex bool
+	withdrawalsResp, err := rp.GetLatestBlockWithdrawals()
+	if err != nil {
+		fmt.Printf("Warning: could not fetch latest beacon block withdrawals: %s\n\n", err.Error())
+	} else if len(withdrawalsResp.Withdrawals) == 0 {
+		fmt.Printf("Latest beacon block (slot %d, exec block %d) has no validator withdrawals.\n\n",
+			withdrawalsResp.Slot, withdrawalsResp.BlockNumber)
+	} else {
+		indexes := make([]string, 0, len(withdrawalsResp.Withdrawals))
+		seen := make(map[string]struct{}, len(withdrawalsResp.Withdrawals))
+		for _, wd := range withdrawalsResp.Withdrawals {
+			if idx, perr := strconv.ParseUint(wd.ValidatorIndex, 10, 64); perr == nil {
+				if !hasLastWithdrawnIndex || idx > lastWithdrawnIndex {
+					lastWithdrawnIndex = idx
+					hasLastWithdrawnIndex = true
+				}
+			}
+			if _, ok := seen[wd.ValidatorIndex]; ok {
+				continue
+			}
+			seen[wd.ValidatorIndex] = struct{}{}
+			indexes = append(indexes, wd.ValidatorIndex)
+		}
+		fmt.Printf("Latest beacon block (slot %d, exec block %d) processed withdrawals for %d validator(s):\n",
+			withdrawalsResp.Slot, withdrawalsResp.BlockNumber, len(indexes))
+		fmt.Printf("  %s\n\n", strings.Join(indexes, ", "))
+	}
+	var estimate api.BeaconWithdrawalQueueEstimateResponse
+	if fetchExitQueueEstimate {
+		// Print an estimate of the beacon chain withdrawal queue time
+		fmt.Println("Fetching beacon chain exit queue estimate... This may take a while...")
+		if estimate, err = rp.GetBeaconWithdrawalQueueEstimate(); err != nil {
+			fmt.Printf("Warning: could not fetch beacon chain exit queue estimate: %s\n\n", err.Error())
+		} else if estimate.ExitQueueGwei == 0 {
+			fmt.Println("The beacon chain exit queue is currently empty.")
+			fmt.Printf("At the current churn limit of %s ETH/epoch, a new exit request would be processed in the next epoch (~%s).\n\n",
+				formatGwei(estimate.ChurnPerEpochGwei), (time.Duration(estimate.SecondsPerEpoch) * time.Second).Round(time.Second))
+		} else {
+			wait := formatDaysHours(time.Duration(estimate.EstimatedQueueSeconds) * time.Second)
+			fmt.Printf("Beacon chain exit queue: %s ETH waiting to exit.\n",
+				formatGwei(estimate.ExitQueueGwei))
+			fmt.Printf("Churn limit: %s ETH/epoch -> estimated %d epochs (~%s) to process the queue.\n\n",
+				formatGwei(estimate.ChurnPerEpochGwei), estimate.EstimatedQueueEpochs, wait)
+		}
+	} else {
+		fmt.Println("Skipping the beacon chain exit queue estimate. Use the --fetch-estimate flag to fetch it.")
+		fmt.Println()
+	}
+
 	// Get Megapool status
 	status, err := rp.MegapoolStatus(false)
 	if err != nil {
 		return 0, false, err
 	}
 
 	activeValidators := []api.MegapoolValidatorDetails{}
+	exitingValidators := []api.MegapoolValidatorDetails{}
 
 	for _, validator := range status.Megapool.Validators {
-		if validator.Activated && !validator.Exiting && !validator.Exited {
+		if validator.Activated && !validator.Exiting && !validator.Exited && validator.BeaconStatus.Status != beacon.ValidatorState_ActiveExiting {
 			// Check if validator is old enough to exit
 			earliestExitEpoch := validator.BeaconStatus.ActivationEpoch + 256
 			if status.BeaconHead.Epoch >= earliestExitEpoch {
 				activeValidators = append(activeValidators, validator)
 			}
 		}
+		if validator.BeaconStatus.Status == beacon.ValidatorState_ActiveExiting {
+			exitingValidators = append(exitingValidators, validator)
+		}
+	}
+
+	// Print exiting validators
+	if len(exitingValidators) > 0 {
+		sortExitingValidatorsBySweep(exitingValidators, lastWithdrawnIndex, hasLastWithdrawnIndex)
+		fmt.Println("The following validators are still active and have already received their exit request on the Beacon Chain:")
+		for _, v := range exitingValidators {
+			fmt.Printf("ID %d: - Index %d Pubkey: 0x%s\n", v.ValidatorId, v.ValidatorIndex, v.PubKey.String())
+		}
+		fmt.Println()
 	}
 	if len(activeValidators) > 0 {
 		sort.Sort(ByIndex(activeValidators))

rocketpool-cli/megapool/exit-validator_test.go

@@ -0,0 +1,153 @@
+package megapool
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/rocket-pool/smartnode/shared/types/api"
+)
+
+// makeValidators builds a slice of MegapoolValidatorDetails with only the
+// fields that sortExitingValidatorsBySweep cares about populated. ValidatorId
+// is set to a distinct value derived from the index so we can also assert that
+// the original elements (not just their indices) are preserved.
+func makeValidators(indices ...uint64) []api.MegapoolValidatorDetails {
+	out := make([]api.MegapoolValidatorDetails, 0, len(indices))
+	for i, idx := range indices {
+		out = append(out, api.MegapoolValidatorDetails{
+			ValidatorId:    uint32(1000 + i),
+			ValidatorIndex: idx,
+		})
+	}
+	return out
+}
+
+func indexesOf(validators []api.MegapoolValidatorDetails) []uint64 {
+	out := make([]uint64, len(validators))
+	for i, v := range validators {
+		out[i] = v.ValidatorIndex
+	}
+	return out
+}
+
+func TestSortExitingValidatorsBySweep(t *testing.T) {
+	tests := []struct {
+		name                  string
+		input                 []uint64
+		lastWithdrawnIndex    uint64
+		hasLastWithdrawnIndex bool
+		want                  []uint64
+	}{
+		{
+			name:                  "empty slice does not panic",
+			input:                 []uint64{},
+			lastWithdrawnIndex:    100,
+			hasLastWithdrawnIndex: true,
+			want:                  []uint64{},
+		},
+		{
+			name:                  "single validator above pivot",
+			input:                 []uint64{200},
+			lastWithdrawnIndex:    100,
+			hasLastWithdrawnIndex: true,
+			want:                  []uint64{200},
+		},
+		{
+			name:                  "single validator below pivot",
+			input:                 []uint64{50},
+			lastWithdrawnIndex:    100,
+			hasLastWithdrawnIndex: true,
+			want:                  []uint64{50},
+		},
+		{
+			name:                  "no pivot falls back to ascending order",
+			input:                 []uint64{300, 50, 200, 100},
+			hasLastWithdrawnIndex: false,
+			want:                  []uint64{50, 100, 200, 300},
+		},
+		{
+			name:                  "pivot in the middle splits and sorts each half",
+			input:                 []uint64{300, 50, 200, 100, 400, 25},
+			lastWithdrawnIndex:    150,
+			hasLastWithdrawnIndex: true,
+			want:                  []uint64{200, 300, 400, 25, 50, 100},
+		},
+		{
+			name:                  "pivot equal to a validator index puts that validator after the wrap",
+			input:                 []uint64{100, 50, 150, 200},
+			lastWithdrawnIndex:    100,
+			hasLastWithdrawnIndex: true,
+			want:                  []uint64{150, 200, 50, 100},
+		},
+		{
+			name:                  "pivot above all validators wraps everyone (plain ascending)",
+			input:                 []uint64{50, 30, 10, 20},
+			lastWithdrawnIndex:    1000,
+			hasLastWithdrawnIndex: true,
+			want:                  []uint64{10, 20, 30, 50},
+		},
+		{
+			name:                  "pivot below all validators leaves them all 'after' (plain ascending)",
+			input:                 []uint64{50, 30, 10, 20},
+			lastWithdrawnIndex:    0,
+			hasLastWithdrawnIndex: true,
+			want:                  []uint64{10, 20, 30, 50},
+		},
+		{
+			name:                  "already in sweep order is unchanged",
+			input:                 []uint64{200, 300, 400, 25, 50, 100},
+			lastWithdrawnIndex:    150,
+			hasLastWithdrawnIndex: true,
+			want:                  []uint64{200, 300, 400, 25, 50, 100},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			validators := makeValidators(tt.input...)
+
+			sortExitingValidatorsBySweep(validators, tt.lastWithdrawnIndex, tt.hasLastWithdrawnIndex)
+
+			got := indexesOf(validators)
+			// Full struct is unreadable in %v; print index order and compact id@index chain.
+			t.Logf("final sweep order (validator indices): %v", got)
+			parts := make([]string, len(validators))
+			for i, v := range validators {
+				parts[i] = fmt.Sprintf("%d@%d", v.ValidatorId, v.ValidatorIndex)
+			}
+			idIndexLine := strings.Join(parts, " → ")
+			if idIndexLine == "" {
+				idIndexLine = "(empty)"
+			}
+			t.Logf("final sweep order (id@index): %s", idIndexLine)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Fatalf("unexpected order: got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+// TestSortExitingValidatorsBySweepPreservesElements confirms that the helper
+// reorders the original elements (preserving fields like ValidatorId) rather
+// than producing copies that drop unrelated data.
+func TestSortExitingValidatorsBySweepPreservesElements(t *testing.T) {
+	validators := []api.MegapoolValidatorDetails{
+		{ValidatorId: 11, ValidatorIndex: 300},
+		{ValidatorId: 22, ValidatorIndex: 50},
+		{ValidatorId: 33, ValidatorIndex: 200},
+		{ValidatorId: 44, ValidatorIndex: 100},
+	}
+
+	sortExitingValidatorsBySweep(validators, 150, true)
+
+	wantIndexes := []uint64{200, 300, 50, 100}
+	wantIDs := []uint32{33, 11, 22, 44}
+	for i, v := range validators {
+		if v.ValidatorIndex != wantIndexes[i] || v.ValidatorId != wantIDs[i] {
+			t.Fatalf("position %d: got (id=%d, index=%d), want (id=%d, index=%d)",
+				i, v.ValidatorId, v.ValidatorIndex, wantIDs[i], wantIndexes[i])
+		}
+	}
+}