feat(dist): add structured logging to dist backend (Phase A.1)

Introduce `WithDistLogger(*slog.Logger)` option for `DistMemory`. When
supplied, the dist backend emits structured `slog` records for all
operational events across its background loops and error surfaces:

- HTTP listener bind success and failure
- Serve-goroutine exits (previously silent in `serveErr` only)
- Peer state transitions: suspect-on-timeout, suspect-on-probe-failure,
  and dead-peer pruning
- Merkle sync fetch failures
- Rebalance migration forward failures
- Hint dropped after replay error

All records are pre-bound with `component=dist_memory` and
`node_id=<id>` at construction so operators can grep/filter without
each call site re-attaching the node identity.

Default behaviour is unchanged: the zero-value path installs a
`slog.DiscardHandler` so library code never writes to stderr unless the
caller opts in.

Add `tests/dist_logging_test.go` with two contract tests:
- `TestDistMemory_LoggerEmitsListenerStart` — asserts the listener-start
  record is emitted with the expected level and attributes.
- `TestDistMemory_LoggerDefaultIsSilent` — asserts no-option construction
  still succeeds without any side effects.

Update `.mdl_style.rb` to allow `MD024` (`allow_different_nesting`) so
the Keep-a-Changelog heading structure passes linting.

Update `CHANGELOG.md`: add `[Unreleased]` entry for this work; correct
version numbers from `2.0.x` to `0.5.0`/`0.4.3`; fix comparison links
to match actual tags.

Author: hyp3rd

.mdl_style.rb

@@ -5,3 +5,11 @@
 rule 'MD007', :indent => 3
 
 rule "MD029", style => "one"
+
+# Keep-a-Changelog (https://keepachangelog.com) uses repeated `### Added`,
+# `### Fixed`, `### Security` headings under each `## [version]` heading by
+# design. MD024 with the default config flags those as duplicates.
+# allow_different_nesting permits same-text headings as long as they sit
+# under distinct parent headings — which is exactly the Keep-a-Changelog
+# shape, and still catches genuine duplicates within the same section.
+rule "MD024", :allow_different_nesting => true

CHANGELOG.md

@@ -6,7 +6,20 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
-## [2.0.1] — 2026-05-05
+### Added
+
+- **Structured logging on the dist backend.** New `WithDistLogger(*slog.Logger)`
+  option wires a structured logger into the dist backend's background
+  loops (heartbeat, hint replay, rebalance, merkle sync) and operational
+  error surfaces (HTTP listener bind failures, serve-goroutine exits,
+  failed migrations during rebalance, dropped hints, peer state
+  transitions). Library default is silent — `WithDistLogger` not called
+  installs a `slog.DiscardHandler` so the dist backend never writes to
+  stderr unless the caller opts in. Every record is pre-bound with
+  `component=dist_memory` and `node_id=<id>` attributes for grep/filter.
+  Phase A.1 of the production-readiness work.
+
+## [0.5.0] — 2026-05-05
 
 ### Security
 
@@ -25,7 +38,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
   in via the new `DistHTTPAuth.AllowAnonymousInbound` field. All other
   configurations (`Token`-only, `Token+ServerVerify`, `Token+ClientSign`,
   `ServerVerify`-only) are unaffected. Reported by the post-tag
-  security review; addressed before any v2.0.0 public announcement.
+  security review; addressed before any v0.5.0 public announcement.
 
 ### Added
 
@@ -34,7 +47,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - `sentinel.ErrInsecureAuthConfig` — surfaced from `NewDistMemory` when
   the auth policy would silently disable inbound enforcement.
 
-## [2.0.0] — 2026-05-04
+## [0.4.3] — 2026-05-04
 
 A modernization release. The headline themes:
 
@@ -86,7 +99,6 @@ RFCs that informed the design decisions live under [docs/rfcs/](docs/rfcs/).
 ### Performance
 
 Measurements on Apple M4 Pro, `go test -bench`, `count=5`, benchstat.
-Full release snapshot captured in [bench-v2.0.0.txt](bench-v2.0.0.txt).
 
 - **Per-shard atomic `Count`.** `BenchmarkConcurrentMap_Count`:
   53 → ~10 ns/op. `_CountParallel`: 1181 → ~13 ns/op. Eliminates the
@@ -186,5 +198,5 @@ Worth surfacing for contributors:
   [RFC document](docs/rfcs/0001-backend-owned-eviction.md) preserves
   the measurement and the lessons.
 
-[Unreleased]: https://github.com/hyp3rd/hypercache/compare/v2.0.0...HEAD
-[2.0.0]: https://github.com/hyp3rd/hypercache/releases/tag/v2.0.0
+Unreleased: <https://github.com/hyp3rd/hypercache/compare/v0.5.0...HEAD>
+Released: [0.5.0](https://github.com/hyp3rd/hypercache/releases/tag/v0.5.0)

pkg/backend/dist_http_server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/subtle"
 	"crypto/tls"
+	"log/slog"
 	"net"
 	"net/http"
 	"strconv"
@@ -47,6 +48,12 @@ type distHTTPServer struct {
 	// use, TLS handshake failure on accept) instead of having them
 	// silently swallowed.
 	serveErr atomic.Pointer[error]
+	// logger is the structured logger inherited from the parent
+	// DistMemory. Used to surface serve-goroutine errors that previously
+	// only landed in serveErr (LastServeError accessor) — operators
+	// running with a configured logger now see them in their log stream
+	// at the moment of failure, not just on demand.
+	logger *slog.Logger
 }
 
 // DistHTTPAuth configures authentication for the dist HTTP server
@@ -482,8 +489,17 @@ func (s *distHTTPServer) listen(ctx context.Context) error {
 		if serveErr != nil {
 			// Stash so operators can read it via LastServeError(); a
 			// listener that crashed silently is the worst kind of
-			// production bug.
+			// production bug. Also surface to the structured logger when
+			// configured so the failure shows up in the operator's log
+			// stream at the moment it happens, not just on demand.
 			s.serveErr.Store(&serveErr)
+
+			if s.logger != nil {
+				s.logger.Error("dist HTTP serve goroutine exited",
+					slog.String("addr", s.addr),
+					slog.Any("err", serveErr),
+				)
+			}
 		}
 	}()
 

pkg/backend/dist_memory.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"hash"
 	"hash/fnv"
+	"log/slog"
 	"math/big"
 	"slices"
 	"strings"
@@ -157,6 +158,16 @@ type DistMemory struct {
 
 	// stopped guards Stop() against double-invocation (idempotent shutdown).
 	stopped atomic.Bool
+
+	// logger is the structured logger used by background loops
+	// (heartbeat, hint replay, rebalance, gossip, merkle sync) and
+	// error surfaces (transport bind failures, sync errors, dropped
+	// hints). Defaults to a no-op handler writing to io.Discard so
+	// library code does not write to stderr unless the caller opts
+	// in via WithDistLogger. All log lines are pre-bound with
+	// `node_id` so operators can grep/filter without the call sites
+	// having to weave the ID through every record.
+	logger *slog.Logger
 }
 
 const (
@@ -433,6 +444,11 @@ func (dm *DistMemory) SyncWith(ctx context.Context, nodeID string) error {
 
 	remoteTree, err := transport.FetchMerkle(ctx, nodeID)
 	if err != nil {
+		dm.logger.Warn("merkle sync fetch failed",
+			slog.String("peer_id", nodeID),
+			slog.Any("err", err),
+		)
+
 		return err
 	}
 
@@ -625,6 +641,29 @@ func WithDistHTTPAuth(auth DistHTTPAuth) DistMemoryOption {
 	return func(dm *DistMemory) { dm.httpAuth = auth }
 }
 
+// WithDistLogger supplies a structured logger for the dist backend's
+// background loops (heartbeat, hint replay, rebalance, gossip, merkle
+// auto-sync) and operational error surfaces (HTTP listener failures,
+// transport errors, dropped hints). The supplied logger is wrapped with
+// `node_id` and `component=dist_memory` attributes before use, so call
+// sites do not need to weave the node ID through every record.
+//
+// Pass slog.Default() to inherit the application's logger, or supply a
+// custom *slog.Logger with the desired level / handler. Zero-value (no
+// option call) keeps the dist backend silent — the default uses an
+// io.Discard handler, which means library code never writes to stderr
+// unless the caller opts in.
+//
+// nil is treated as "no change" — useful when callers conditionally
+// build options.
+func WithDistLogger(logger *slog.Logger) DistMemoryOption {
+	return func(dm *DistMemory) {
+		if logger != nil {
+			dm.logger = logger
+		}
+	}
+}
+
 // NewDistMemory creates a new DistMemory backend.
 func NewDistMemory(ctx context.Context, opts ...DistMemoryOption) (IBackend[DistMemory], error) {
 	// Derive a server-lifetime context from the caller's ctx so that:
@@ -659,6 +698,20 @@ func NewDistMemory(ctx context.Context, opts ...DistMemoryOption) (IBackend[Dist
 		return nil, authErr
 	}
 
+	// Default the logger to a no-op handler so library code never writes
+	// to stderr unless the caller opts in via WithDistLogger. Bind the
+	// node identity once here so call sites can log without re-attaching
+	// it on every record. Done before subsystem startup so background
+	// loops capture the bound logger when they spawn.
+	if dm.logger == nil {
+		dm.logger = slog.New(slog.DiscardHandler)
+	}
+
+	dm.logger = dm.logger.With(
+		slog.String("component", "dist_memory"),
+		slog.String("node_id", dm.nodeID),
+	)
+
 	dm.ensureShardConfig()
 	dm.initMembershipIfNeeded()
 	// Pass the lifecycle ctx to subsystems that capture it (HTTP handlers,
@@ -1917,7 +1970,18 @@ func (dm *DistMemory) migrateIfNeeded(ctx context.Context, item *cache.Item) {
 	dm.metrics.rebalancedKeys.Add(1)
 	dm.metrics.rebalancedPrimary.Add(1)
 
-	_ = transport.ForwardSet(ctx, string(owners[0]), item, true)
+	// Fire-and-forget forwarding: failures are dropped silently today
+	// (Phase B will introduce a retry queue). Logging is the minimum
+	// surface so operators can correlate vanished keys with transport
+	// failures during rolling deploys.
+	migrationErr := transport.ForwardSet(ctx, string(owners[0]), item, true)
+	if migrationErr != nil {
+		dm.logger.Warn("rebalance migration forward failed",
+			slog.String("key", item.Key),
+			slog.String("new_primary", string(owners[0])),
+			slog.Any("err", migrationErr),
+		)
+	}
 
 	// Update originalPrimary so we don't recount repeatedly.
 	sh := dm.shardFor(item.Key)
@@ -2077,14 +2141,26 @@ func (dm *DistMemory) tryStartHTTP(ctx context.Context) {
 	server := newDistHTTPServer(dm.nodeAddr, limits, dm.httpAuth)
 
 	server.ctx = dm.lifeCtx // handler-side cancellation tied to Stop
+	server.logger = dm.logger
 
 	err := server.start(ctx, dm)
-	if err != nil { // best-effort
+	if err != nil { // best-effort, but the operator must see this
+		dm.logger.Error("dist HTTP listener bind failed",
+			slog.String("addr", dm.nodeAddr),
+			slog.Any("err", err),
+		)
+
 		return
 	}
 
 	dm.httpServer = server
 
+	dm.logger.Info("dist HTTP listener started",
+		slog.String("addr", dm.nodeAddr),
+		slog.Bool("tls", limits.TLSConfig != nil),
+		slog.Bool("auth", dm.httpAuth.inboundConfigured()),
+	)
+
 	resolver := dm.makePeerURLResolver(limits)
 
 	dm.storeTransport(NewDistHTTPTransportWithAuth(limits, dm.httpAuth, resolver))
@@ -2609,6 +2685,12 @@ func (dm *DistMemory) processHint(ctx context.Context, nodeID string, entry hint
 
 	dm.metrics.hintedDropped.Add(1)
 
+	dm.logger.Warn("hint dropped after replay error",
+		slog.String("peer_id", nodeID),
+		slog.String("key", entry.item.Key),
+		slog.Any("err", err),
+	)
+
 	return 1
 }
 
@@ -3153,6 +3235,12 @@ func (dm *DistMemory) evaluateLiveness(ctx context.Context, now time.Time, node
 		if dm.membership.Remove(node.ID) {
 			dm.metrics.nodesRemoved.Add(1)
 			dm.metrics.nodesDead.Add(1)
+
+			dm.logger.Warn("peer pruned (dead)",
+				slog.String("peer_id", string(node.ID)),
+				slog.String("peer_addr", node.Address),
+				slog.Duration("elapsed_since_seen", elapsed),
+			)
 		}
 
 		return
@@ -3161,6 +3249,11 @@ func (dm *DistMemory) evaluateLiveness(ctx context.Context, now time.Time, node
 	if dm.hbSuspectAfter > 0 && elapsed > dm.hbSuspectAfter && node.State == cluster.NodeAlive { // suspect
 		dm.membership.Mark(node.ID, cluster.NodeSuspect)
 		dm.metrics.nodesSuspect.Add(1)
+
+		dm.logger.Info("peer marked suspect (timeout)",
+			slog.String("peer_id", string(node.ID)),
+			slog.Duration("elapsed_since_seen", elapsed),
+		)
 	}
 
 	transport := dm.loadTransport()
@@ -3179,6 +3272,11 @@ func (dm *DistMemory) evaluateLiveness(ctx context.Context, now time.Time, node
 		if node.State == cluster.NodeAlive { // escalate
 			dm.membership.Mark(node.ID, cluster.NodeSuspect)
 			dm.metrics.nodesSuspect.Add(1)
+
+			dm.logger.Info("peer marked suspect (probe failed)",
+				slog.String("peer_id", string(node.ID)),
+				slog.Any("err", err),
+			)
 		}
 
 		return

pkg/backend/dist_memory_test_helpers.go

@@ -31,6 +31,7 @@ func (dm *DistMemory) EnableHTTPForTest(ctx context.Context) {
 	server := newDistHTTPServer(dm.nodeAddr, limits, dm.httpAuth)
 
 	server.ctx = dm.lifeCtx // handler-side cancellation tied to Stop
+	server.logger = dm.logger
 
 	err := server.start(ctx, dm)
 	if err != nil {