Observer: add jitter to each monitor to prevent stampedes (#8872)

Update monitor.start() to do a random sleep before probing for the first
time, and to do a 20% jitter on its sleep time between each probe. This
20% jitter matches the 20% used in core.RetryBackoff.

While we're at it, update start() to take a context and listen for
context cancellation, to allow for nicer cleanup at shutdown.

Fixes #8871

Author: aarongable

observer/monitor.go

@@ -2,6 +2,7 @@ package observer
 
 import (
 	"context"
+	"math/rand/v2"
 	"strconv"
 	"time"
 
@@ -14,10 +15,19 @@ type monitor struct {
 	prober probers.Prober
 }
 
-// start spins off a 'Prober' goroutine on an interval of `m.period`
-// with a timeout of half `m.period`
-func (m monitor) start(logger blog.Logger) {
-	ticker := time.NewTicker(m.period)
+// start spins off a 'Prober' goroutine approximately once per `m.period`,
+// with a timeout of half `m.period`. The probe attempts start after a random
+// delay and have 20% jitter around the configured period, to prevent many
+// monitors with the same period from all waking up at the same time.
+func (m monitor) start(ctx context.Context, logger blog.Logger) {
+	// Wait a random duration of at most one period before the first probe,
+	// so that monitors don't all fire at once when the process starts.
+	select {
+	case <-ctx.Done():
+		return
+	case <-time.After(rand.N(m.period)):
+	}
+
 	for {
 		go func() {
 			ctx, cancel := context.WithTimeout(context.Background(), m.period/2)
@@ -42,6 +52,13 @@ func (m monitor) start(logger blog.Logger) {
 					m.prober.Kind(), err == nil, dur.Seconds(), m.prober.Name())
 			}
 		}()
-		<-ticker.C
+
+		// This jitter is equivalent to 1 +/- 0.2*rand.Float64().
+		jitter := 0.8 + 0.4*rand.Float64()
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(time.Duration(float64(m.period) * jitter)):
+		}
 	}
 }

observer/observer.go

@@ -22,10 +22,14 @@ type Observer struct {
 
 // Start spins off a goroutine for each monitor, and waits for a signal to exit
 func (o *Observer) Start() {
+	defer o.shutdown(context.Background())
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	for _, mon := range o.monitors {
-		go mon.start(o.logger)
+		go mon.start(ctx, o.logger)
 	}
 
-	defer o.shutdown(context.Background())
 	cmd.WaitForSignal()
 }