admin: integrate LeaderForwarder into dynamo handler (Task #26 phase 2)

Wire the follower-side forwarder from phase 1 into the dynamo HTTP
handler so writes hitting a follower transparently dispatch to the
leader. Acceptance criteria 2 + 3 from design Section 3.3.2.

Changes:
- DynamoHandler gains a forwarder field plus
  WithLeaderForwarder option. nil keeps the previous "503
  leader_unavailable" behaviour, so a build that ships only the
  leader can still deploy unchanged.
- handleCreate / handleDelete catch ErrTablesNotLeader from the
  source and route to the forwarder. The principal already in
  context flows through to the leader's audit log unchanged.
- writeForwardResult re-emits the leader's structured response
  verbatim — status, payload, and content-type all come from the
  gRPC response so a forwarded request looks identical to a
  leader-direct call from the SPA's perspective.
- writeForwardFailure handles the two non-success paths:
  ErrLeaderUnavailable (election in flight, criterion 3) and
  generic gRPC transport errors. Both produce 503 + Retry-After: 1
  so the SPA's retry policy is uniform regardless of where in the
  chain the 503 came from. ErrLeaderUnavailable is NOT logged as
  an error since elections are routine; transport errors are
  logged at LevelError so operators can investigate.
- A 503 surfacing via the leader's structured response (e.g.,
  leader stepped down mid-request) also gets Retry-After: 1
  added on the way out so the wire contract is the same.

Tests cover criteria 2 (transparent forwarding for both create
and delete), 3 (election-period 503 + Retry-After), the
leader-direct error pass-through (409 conflict), the no-forwarder
fallback, and confirm the role / body validations short-circuit
*before* the forwarder is invoked. Stub LeaderForwarder + a
not-leader source make the tests deterministic without spinning
up a real Raft cluster.

Author: bootjp

internal/admin/dynamo_handler.go

@@ -168,15 +168,22 @@ func (e *ValidationError) Error() string {
 // — the JWT freezes the role at login time, and tokens last one
 // hour. Codex P1 on PR #635 flagged the gap on the HTTP path;
 // the forward server already does this re-evaluation on its side.
+//
+// When the source returns ErrTablesNotLeader and a LeaderForwarder
+// is configured, write requests are forwarded to the leader
+// transparently — the SPA sees a leader-direct response shape
+// regardless of which node it hit (design Section 3.3 criterion 2).
 type DynamoHandler struct {
-	source TablesSource
-	roles  RoleStore
-	logger *slog.Logger
+	source    TablesSource
+	roles     RoleStore
+	forwarder LeaderForwarder
+	logger    *slog.Logger
 }
 
 // NewDynamoHandler binds the source and seeds logging with
-// slog.Default(). Use WithLogger to attach a tagged logger and
-// WithRoleStore to plug in the live access-key role lookup.
+// slog.Default(). Use WithLogger to attach a tagged logger,
+// WithRoleStore to plug in the live access-key role lookup, and
+// WithLeaderForwarder to plug in the follower→leader forwarder.
 func NewDynamoHandler(source TablesSource) *DynamoHandler {
 	return &DynamoHandler{source: source, logger: slog.Default()}
 }
@@ -201,6 +208,16 @@ func (h *DynamoHandler) WithRoleStore(r RoleStore) *DynamoHandler {
 	return h
 }
 
+// WithLeaderForwarder enables transparent follower→leader
+// forwarding. Without it, write requests on a follower fall back
+// to the standard 503 leader_unavailable response. Production
+// wires this to the gRPCForwardClient in main_admin.go; tests
+// inject a stub.
+func (h *DynamoHandler) WithLeaderForwarder(f LeaderForwarder) *DynamoHandler {
+	h.forwarder = f
+	return h
+}
+
 // ServeHTTP routes /tables and /tables/{name}. We do not use
 // http.ServeMux because the admin router already guards the
 // /admin/api/v1/* prefix — adding another mux here would just
@@ -313,12 +330,48 @@ func (h *DynamoHandler) handleCreate(w http.ResponseWriter, r *http.Request) {
 	}
 	summary, err := h.source.AdminCreateTable(r.Context(), principal, body)
 	if err != nil {
+		// On a follower, the source returns ErrTablesNotLeader. If
+		// a forwarder is wired, dispatch to the leader and re-emit
+		// the leader's response verbatim — the SPA cannot tell the
+		// difference between a leader-direct call and a forwarded
+		// one. Without a forwarder, fall through to the standard
+		// 503 leader_unavailable response.
+		if h.tryForwardCreate(w, r, principal, body, err) {
+			return
+		}
 		h.writeTablesError(w, r, "create", err)
 		return
 	}
 	writeAdminJSONStatus(w, r.Context(), h.logger, http.StatusCreated, summary)
 }
 
+// tryForwardCreate handles the follower→leader forwarding path
+// for POST /tables. Returns true when the response has been
+// written (regardless of forward success/failure); the caller
+// should then return without further processing.
+//
+// The "fall through to 503" path runs only when:
+//   - the source error is something other than ErrTablesNotLeader,
+//   - or no LeaderForwarder was configured,
+//   - or the forwarder itself returned ErrLeaderUnavailable
+//     (election in progress on the leader, criterion 3).
+//
+// Any other forwarder failure (gRPC transport error, etc.) is
+// also surfaced as 503 + Retry-After: 1 so the SPA can re-issue.
+// We log the raw error for operators and never echo it to clients.
+func (h *DynamoHandler) tryForwardCreate(w http.ResponseWriter, r *http.Request, principal AuthPrincipal, body CreateTableRequest, sourceErr error) bool {
+	if !errors.Is(sourceErr, ErrTablesNotLeader) || h.forwarder == nil {
+		return false
+	}
+	res, err := h.forwarder.ForwardCreateTable(r.Context(), principal, body)
+	if err != nil {
+		h.writeForwardFailure(w, r, "create", err)
+		return true
+	}
+	h.writeForwardResult(w, r, res)
+	return true
+}
+
 // handleDelete is the DELETE /tables/{name} handler. Success is
 // 204 No Content; the body is intentionally empty so the SPA can
 // treat both 200 and 204 as success without parsing.
@@ -332,6 +385,9 @@ func (h *DynamoHandler) handleDelete(w http.ResponseWriter, r *http.Request, nam
 		return
 	}
 	if err := h.source.AdminDeleteTable(r.Context(), principal, name); err != nil {
+		if h.tryForwardDelete(w, r, principal, name, err) {
+			return
+		}
 		h.writeTablesError(w, r, "delete", err)
 		return
 	}
@@ -390,6 +446,66 @@ func (h *DynamoHandler) principalForWrite(w http.ResponseWriter, r *http.Request
 	return principal, true
 }
 
+// tryForwardDelete is the DELETE counterpart of tryForwardCreate.
+// Same semantics: only the ErrTablesNotLeader source error
+// triggers forwarding, and only when a forwarder is configured.
+func (h *DynamoHandler) tryForwardDelete(w http.ResponseWriter, r *http.Request, principal AuthPrincipal, name string, sourceErr error) bool {
+	if !errors.Is(sourceErr, ErrTablesNotLeader) || h.forwarder == nil {
+		return false
+	}
+	res, err := h.forwarder.ForwardDeleteTable(r.Context(), principal, name)
+	if err != nil {
+		h.writeForwardFailure(w, r, "delete", err)
+		return true
+	}
+	h.writeForwardResult(w, r, res)
+	return true
+}
+
+// writeForwardResult re-emits the leader's structured response
+// verbatim. Status, payload, and content-type all come from the
+// gRPC response so a forwarded request looks identical to a
+// leader-direct call from the SPA's point of view.
+func (h *DynamoHandler) writeForwardResult(w http.ResponseWriter, r *http.Request, res *ForwardResult) {
+	w.Header().Set("Content-Type", res.ContentType)
+	w.Header().Set("Cache-Control", "no-store")
+	// 503 from the leader (e.g. it stepped down mid-request) must
+	// carry Retry-After so the client retries; preserve the
+	// criterion-3 contract on the wire whether the 503 originated
+	// here or at the leader.
+	if res.StatusCode == http.StatusServiceUnavailable {
+		w.Header().Set("Retry-After", "1")
+	}
+	w.WriteHeader(res.StatusCode)
+	if len(res.Payload) > 0 {
+		if _, err := w.Write(res.Payload); err != nil {
+			h.logger.LogAttrs(r.Context(), slog.LevelWarn, "admin forward response write failed",
+				slog.String("error", err.Error()),
+			)
+		}
+	}
+}
+
+// writeForwardFailure handles forwarder errors that did not
+// produce a structured leader response: ErrLeaderUnavailable
+// (election in flight) and gRPC transport errors. Both surface as
+// 503 + Retry-After: 1 — the SPA's retry contract is identical
+// regardless of whether the leader is briefly absent or the
+// network hiccupped.
+func (h *DynamoHandler) writeForwardFailure(w http.ResponseWriter, r *http.Request, op string, err error) {
+	if !errors.Is(err, ErrLeaderUnavailable) {
+		// Not the "no leader known" case — log the raw error so
+		// operators can investigate. Client still sees the same
+		// 503 + Retry-After so they retry uniformly.
+		h.logger.LogAttrs(r.Context(), slog.LevelError, "admin dynamo "+op+" forward failed",
+			slog.String("error", err.Error()),
+		)
+	}
+	w.Header().Set("Retry-After", "1")
+	writeJSONError(w, http.StatusServiceUnavailable, "leader_unavailable",
+		"raft leader currently unavailable; retry shortly")
+}
+
 // writeTablesError translates a TablesSource error into the
 // appropriate HTTP response. Internal-server-error fallthrough logs
 // the raw err.Error() but never sends it to the client, matching