Feat/detailed coordination window diagnostics (#3861)

# Detailed Coordination Window Diagnostics

## Summary

This PR introduces comprehensive diagnostics and metrics tracking for
tBTC coordination windows, significantly enhancing observability into
the coordination process. The changes add detailed per-window and
per-wallet metrics, improve network diagnostics, and expand performance
monitoring capabilities.

A new comprehensive metrics tracking system for coordination windows
that provides:

- **Per-Window Tracking**: Each coordination window is tracked with:
  - Window identification (index, coordination block)
  - Timing information (start time, end time, duration, block ranges)
- Coordination statistics (wallets coordinated, successful/failed
counts)
  - Leader distribution across wallets
  - Action type breakdown
  - Fault statistics (by type and culprit)
  
- **Per-Wallet Details**: For each wallet coordinated in a window:
  - Wallet public key hash
  - Leader address
  - Action type
  - Success/failure status
  - Duration
  - Error messages (if failed)
  - Detailed fault information

- **Memory Management**: Tracks up to 100 recent windows (~25 hours)
with automatic cleanup of older windows to prevent unbounded memory
growth

Author: lionakhnazarov

config/peers_test.go

@@ -24,9 +24,8 @@ func TestResolvePeers(t *testing.T) {
 		"sepolia network": {
 			network: network.Testnet,
 			expectedPeers: []string{
-				"/dns4/bootstrap-0.test.keep.network/tcp/3919/ipfs/16Uiu2HAmCcfVpHwfBKNFbQuhvGuFXHVLQ65gB4sJm7HyrcZuLttH",
-				"/dns4/bootstrap-1.test.keep.network/tcp/3919/ipfs/16Uiu2HAm3eJtyFKAttzJ85NLMromHuRg4yyum3CREMf6CHBBV6KY",
 				"/dns4/bst-a01.test.keep.boar.network/tcp/6001/ipfs/16Uiu2HAmSLDSahiKyTbCNNu8wJmZAsiKF7wuYJ8mogY8ZuAG1jhu",
+				"/dns4/keep-validator-0.eks-ap-northeast-2-secure.staging.staked.cloud/tcp/3919/ipfs/16Uiu2HAm77eSvRq5ioD4J8VFPkq3bJHBEHkssCuiFkgAoABwjo2S",
 			},
 		},
 		"developer network": {

pkg/clientinfo/performance.go

@@ -7,6 +7,12 @@ import (
 	"runtime"
 	"sync"
 	"time"
+
+	// gopsutil provides cross-platform system and process utilities.
+	// It supports linux/amd64 and darwin/amd64 (the target platforms for this codebase),
+	// as well as Windows, FreeBSD, OpenBSD, and Solaris.
+	"github.com/shirou/gopsutil/cpu"
+	"github.com/shirou/gopsutil/mem"
 )
 
 // PerformanceMetricsRecorder provides a simple interface for recording
@@ -107,27 +113,40 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
 		MetricSigningSuccessTotal,
 		MetricSigningFailedTotal,
 		MetricSigningTimeoutsTotal,
+		MetricRedemptionExecutionsTotal,
+		MetricRedemptionExecutionsSuccessTotal,
+		MetricRedemptionExecutionsFailedTotal,
+		MetricRedemptionProofSubmissionsTotal,
+		MetricRedemptionProofSubmissionsSuccessTotal,
+		MetricRedemptionProofSubmissionsFailedTotal,
 		MetricWalletActionsTotal,
 		MetricWalletActionSuccessTotal,
 		MetricWalletActionFailedTotal,
 		MetricWalletHeartbeatFailuresTotal,
 		MetricCoordinationWindowsDetectedTotal,
 		MetricCoordinationProceduresExecutedTotal,
 		MetricCoordinationFailedTotal,
+		MetricCoordinationLeaderTimeoutTotal,
 		MetricPeerConnectionsTotal,
 		MetricPeerDisconnectionsTotal,
 		MetricMessageBroadcastTotal,
 		MetricMessageReceivedTotal,
 		MetricPingTestsTotal,
 		MetricPingTestSuccessTotal,
 		MetricPingTestFailedTotal,
+		MetricNetworkJoinRequestsTotal,
+		MetricNetworkJoinRequestsSuccessTotal,
+		MetricNetworkJoinRequestsFailedTotal,
+		MetricFirewallRejectionsTotal,
 		MetricWalletDispatcherRejectedTotal,
 	}
 
 	// First, initialize all counters in the map
+	pm.countersMutex.Lock()
 	for _, name := range counters {
 		pm.counters[name] = &counter{value: 0}
 	}
+	pm.countersMutex.Unlock()
 
 	// Then, register observers (this prevents concurrent map read/write)
 	for _, name := range counters {
@@ -159,7 +178,9 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
 			WalletActionMetricName(actionType, "failed_total"),
 		}
 		for _, name := range actionCounters {
+			pm.countersMutex.Lock()
 			pm.counters[name] = &counter{value: 0}
+			pm.countersMutex.Unlock()
 			metricName := name // Capture for closure
 			pm.registry.ObserveApplicationSource(
 				"performance",
@@ -181,9 +202,11 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
 
 		// Register duration metric for this action type
 		durationName := WalletActionMetricName(actionType, "duration_seconds")
+		pm.histogramsMutex.Lock()
 		pm.histograms[durationName] = &histogram{
 			buckets: make(map[float64]float64),
 		}
+		pm.histogramsMutex.Unlock()
 		durationMetricName := durationName // Capture for closure
 		pm.registry.ObserveApplicationSource(
 			"performance",
@@ -212,17 +235,22 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
 	durationMetrics := []string{
 		MetricDKGDurationSeconds,
 		MetricSigningDurationSeconds,
+		MetricRedemptionActionDurationSeconds,
 		MetricWalletActionDurationSeconds,
 		MetricCoordinationDurationSeconds,
+		MetricCoordinationWindowDurationSeconds,
 		MetricPingTestDurationSeconds,
+		MetricNetworkHandshakeDurationSeconds,
 	}
 
 	// First, initialize all histograms in the map
+	pm.histogramsMutex.Lock()
 	for _, name := range durationMetrics {
 		pm.histograms[name] = &histogram{
 			buckets: make(map[float64]float64),
 		}
 	}
+	pm.histogramsMutex.Unlock()
 
 	// Then, register observers (this prevents concurrent map read/write)
 	for _, name := range durationMetrics {
@@ -270,12 +298,17 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
 		MetricCPUUtilization,
 		MetricMemoryUsageMB,
 		MetricGoroutineCount,
+		MetricCPULoadPercent,
+		MetricRAMUtilizationPercent,
+		MetricSwapUtilizationPercent,
 	}
 
 	// First, initialize all gauges in the map
+	pm.gaugesMutex.Lock()
 	for _, name := range gauges {
 		pm.gauges[name] = &gauge{value: 0}
 	}
+	pm.gaugesMutex.Unlock()
 
 	// Then, register observers (this prevents concurrent map read/write)
 	for _, name := range gauges {
@@ -394,7 +427,7 @@ func (pm *PerformanceMetrics) SetGauge(name string, value float64) {
 // observeSystemMetrics periodically collects and updates system metrics
 // including CPU utilization, memory usage, and goroutine count.
 func (pm *PerformanceMetrics) observeSystemMetrics(ctx context.Context) {
-	ticker := time.NewTicker(10 * time.Second) // Update every 10 seconds
+	ticker := time.NewTicker(60 * time.Second) // Update every 10 seconds
 	defer ticker.Stop()
 
 	var lastMemStats runtime.MemStats
@@ -428,6 +461,9 @@ func (pm *PerformanceMetrics) observeSystemMetrics(ctx context.Context) {
 				lastMemStats = memStats
 				lastUpdateTime = now
 			}
+
+			// Update OS-level machine stats
+			pm.updateMachineStats()
 		case <-ctx.Done():
 			return
 		}
@@ -483,6 +519,37 @@ func (pm *PerformanceMetrics) calculateCPUUtilizationHeuristic(
 	return cpuUtilization
 }
 
+// updateMachineStats collects and updates OS-level machine statistics
+// including CPU load, RAM utilization, and swapfile utilization.
+func (pm *PerformanceMetrics) updateMachineStats() {
+	// Get CPU load percentage (1-second average)
+	// NOTE: cpu.Percent blocks for the specified duration (1 second) to sample
+	// CPU usage over that interval. This blocking behavior is intentional and
+	// necessary to obtain an accurate CPU utilization measurement. The function
+	// will not return until the 1-second sampling period completes.
+	cpuPercent, err := cpu.Percent(time.Second, false)
+	if err == nil && len(cpuPercent) > 0 {
+		pm.SetGauge(MetricCPULoadPercent, cpuPercent[0])
+	}
+
+	// Get memory statistics
+	memInfo, err := mem.VirtualMemory()
+	if err == nil {
+		// RAM utilization percentage
+		pm.SetGauge(MetricRAMUtilizationPercent, memInfo.UsedPercent)
+
+		// Swap utilization percentage
+		swapInfo, err := mem.SwapMemory()
+		if err == nil && swapInfo.Total > 0 {
+			swapUtilizationPercent := (float64(swapInfo.Used) / float64(swapInfo.Total)) * 100.0
+			pm.SetGauge(MetricSwapUtilizationPercent, swapUtilizationPercent)
+		} else {
+			// If swap is not available or has no total, set to 0
+			pm.SetGauge(MetricSwapUtilizationPercent, 0)
+		}
+	}
+}
+
 // NoOpPerformanceMetrics is a no-op implementation of PerformanceMetricsRecorder
 // that can be used when metrics are disabled.
 type NoOpPerformanceMetrics struct{}
@@ -550,6 +617,17 @@ const (
 	MetricSigningAttemptsPerOperation = "signing_attempts_per_operation"
 	MetricSigningTimeoutsTotal        = "signing_timeouts_total"
 
+	// Redemption Metrics
+	MetricRedemptionExecutionsTotal        = "redemption_executions_total"
+	MetricRedemptionExecutionsSuccessTotal = "redemption_executions_success_total"
+	MetricRedemptionExecutionsFailedTotal  = "redemption_executions_failed_total"
+	MetricRedemptionActionDurationSeconds  = "redemption_action_duration_seconds"
+
+	// Redemption Proof Submission Metrics (SPV maintainer)
+	MetricRedemptionProofSubmissionsTotal        = "redemption_proof_submissions_total"
+	MetricRedemptionProofSubmissionsSuccessTotal = "redemption_proof_submissions_success_total"
+	MetricRedemptionProofSubmissionsFailedTotal  = "redemption_proof_submissions_failed_total"
+
 	// Wallet Action Metrics (aggregate)
 	MetricWalletActionsTotal           = "wallet_actions_total"
 	MetricWalletActionSuccessTotal     = "wallet_action_success_total"
@@ -565,9 +643,18 @@ const (
 	// Coordination Metrics
 	MetricCoordinationWindowsDetectedTotal    = "coordination_windows_detected_total"
 	MetricCoordinationProceduresExecutedTotal = "coordination_procedures_executed_total"
-	MetricCoordinationFailedTotal             = "coordination_failed_total"
+	MetricCoordinationFailedTotal             = "coordination_failed_total"         // Only when node is leader
+	MetricCoordinationLeaderTimeoutTotal      = "coordination_leader_timeout_total" // When follower observes leader timeout
 	MetricCoordinationDurationSeconds         = "coordination_duration_seconds"
 
+	// Coordination Window Metrics (per-window tracking)
+	MetricCoordinationWindowDurationSeconds    = "coordination_window_duration_seconds"
+	MetricCoordinationWindowWalletsCoordinated = "coordination_window_wallets_coordinated"
+	MetricCoordinationWindowWalletsSuccessful  = "coordination_window_wallets_successful"
+	MetricCoordinationWindowWalletsFailed      = "coordination_window_wallets_failed"
+	MetricCoordinationWindowTotalFaults        = "coordination_window_total_faults"
+	MetricCoordinationWindowCoordinationBlock  = "coordination_window_coordination_block"
+
 	// Network Metrics
 	MetricIncomingMessageQueueSize = "incoming_message_queue_size"
 	MetricMessageHandlerQueueSize  = "message_handler_queue_size"
@@ -580,14 +667,24 @@ const (
 	MetricPingTestFailedTotal      = "ping_test_failed_total"
 	MetricPingTestDurationSeconds  = "ping_test_duration_seconds"
 
+	// Network Join Request Metrics (inbound connection attempts from peers)
+	MetricNetworkJoinRequestsTotal        = "network_join_requests_total"         // Total inbound join attempts
+	MetricNetworkJoinRequestsSuccessTotal = "network_join_requests_success_total" // Successful joins
+	MetricNetworkJoinRequestsFailedTotal  = "network_join_requests_failed_total"  // Failed joins (handshake failure)
+	MetricNetworkHandshakeDurationSeconds = "network_handshake_duration_seconds"  // Handshake duration
+	MetricFirewallRejectionsTotal         = "firewall_rejections_total"           // Firewall rejections
+
 	// Wallet Dispatcher Metrics
 	MetricWalletDispatcherActiveActions = "wallet_dispatcher_active_actions"
 	MetricWalletDispatcherRejectedTotal = "wallet_dispatcher_rejected_total"
 
 	// System Metrics
-	MetricCPUUtilization = "cpu_utilization_percent"
-	MetricMemoryUsageMB  = "memory_usage_mb"
-	MetricGoroutineCount = "goroutine_count"
+	MetricCPUUtilization         = "cpu_utilization_percent"
+	MetricMemoryUsageMB          = "memory_usage_mb"
+	MetricGoroutineCount         = "goroutine_count"
+	MetricCPULoadPercent         = "cpu_load_percent"
+	MetricRAMUtilizationPercent  = "ram_utilization_percent"
+	MetricSwapUtilizationPercent = "swap_utilization_percent"
 )
 
 // WalletActionMetricName generates a metric name for a specific wallet action type.

pkg/clientinfo/performance_test.go

@@ -64,19 +64,40 @@ func TestConcurrentCounterDifferentMetrics(t *testing.T) {
 		MetricSigningFailedTotal,
 	}
 
+	// Add timeout to prevent test from hanging indefinitely
+	testCtx, testCancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer testCancel()
+
+	done := make(chan struct{})
 	var wg sync.WaitGroup
 	for _, metricName := range metrics {
 		for i := 0; i < numGoroutines; i++ {
 			wg.Add(1)
 			go func(name string) {
 				defer wg.Done()
 				for j := 0; j < incrementsPer; j++ {
-					pm.IncrementCounter(name, 1)
+					select {
+					case <-testCtx.Done():
+						return
+					default:
+						pm.IncrementCounter(name, 1)
+					}
 				}
 			}(metricName)
 		}
 	}
-	wg.Wait()
+
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		// Test completed successfully
+	case <-testCtx.Done():
+		t.Fatal("Test timed out waiting for goroutines to complete")
+	}
 
 	expected := float64(numGoroutines * incrementsPer)
 	for _, metricName := range metrics {
@@ -320,6 +341,9 @@ func TestMetricsInitialization(t *testing.T) {
 		MetricCPUUtilization,
 		MetricMemoryUsageMB,
 		MetricGoroutineCount,
+		MetricCPULoadPercent,
+		MetricRAMUtilizationPercent,
+		MetricSwapUtilizationPercent,
 	}
 
 	for _, gaugeName := range gauges {

pkg/maintainer/spv/redemptions.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/keep-network/keep-core/pkg/bitcoin"
+	"github.com/keep-network/keep-core/pkg/clientinfo"
 	"github.com/keep-network/keep-core/pkg/tbtc"
 )
 
@@ -46,12 +47,12 @@ func submitRedemptionProof(
 ) error {
 	// Record proof submission attempt
 	if metricsRecorder != nil {
-		metricsRecorder.IncrementCounter("redemption_proof_submissions_total", 1)
+		metricsRecorder.IncrementCounter(clientinfo.MetricRedemptionProofSubmissionsTotal, 1)
 	}
 
 	if requiredConfirmations == 0 {
 		if metricsRecorder != nil {
-			metricsRecorder.IncrementCounter("redemption_proof_submissions_failed_total", 1)
+			metricsRecorder.IncrementCounter(clientinfo.MetricRedemptionProofSubmissionsFailedTotal, 1)
 		}
 		return fmt.Errorf(
 			"provided required confirmations count must be greater than 0",
@@ -65,7 +66,7 @@ func submitRedemptionProof(
 	)
 	if err != nil {
 		if metricsRecorder != nil {
-			metricsRecorder.IncrementCounter("redemption_proof_submissions_failed_total", 1)
+			metricsRecorder.IncrementCounter(clientinfo.MetricRedemptionProofSubmissionsFailedTotal, 1)
 		}
 		return fmt.Errorf(
 			"failed to assemble transaction spv proof: [%v]",
@@ -79,7 +80,7 @@ func submitRedemptionProof(
 	)
 	if err != nil {
 		if metricsRecorder != nil {
-			metricsRecorder.IncrementCounter("redemption_proof_submissions_failed_total", 1)
+			metricsRecorder.IncrementCounter(clientinfo.MetricRedemptionProofSubmissionsFailedTotal, 1)
 		}
 		return fmt.Errorf(
 			"error while parsing transaction inputs: [%v]",
@@ -94,7 +95,7 @@ func submitRedemptionProof(
 		walletPublicKeyHash,
 	); err != nil {
 		if metricsRecorder != nil {
-			metricsRecorder.IncrementCounter("redemption_proof_submissions_failed_total", 1)
+			metricsRecorder.IncrementCounter(clientinfo.MetricRedemptionProofSubmissionsFailedTotal, 1)
 		}
 		return fmt.Errorf(
 			"failed to submit redemption proof with reimbursement: [%v]",
@@ -104,7 +105,7 @@ func submitRedemptionProof(
 
 	// Record successful proof submission
 	if metricsRecorder != nil {
-		metricsRecorder.IncrementCounter("redemption_proof_submissions_success_total", 1)
+		metricsRecorder.IncrementCounter(clientinfo.MetricRedemptionProofSubmissionsSuccessTotal, 1)
 	}
 
 	return nil