feat(email): failover ESP provider — secondary on primary reject/error (#35) (#93)

Add a FailoverProvider that wraps an ordered chain of EmailProvider
(primary first) and retries the same EventEmail through each secondary
ESP when the primary errors or hard-rejects a send. A send is
"successful" (forwarder claims the row, preserving worker convention 6
claim-after-2xx) iff ANY provider in the chain returns success.

This directly hardens the standing P0 (Brevo sender unvalidated → every
send rejected): wire EMAIL_PROVIDER_FALLBACK=ses and a Brevo
account-level reject falls through to SES on the same payload.

Inert by default: EMAIL_PROVIDER_FALLBACK unset → NewProvider returns
the bare single provider, byte-identical to today (no wrapper, no
metric). Proven by TestFactory_NoFallback_ReturnsBareProvider.

- factory: single provider build factored into buildOne(name, cfg) so
  primary + fallbacks share ONE switch (no duplicated provider switch).
  Unknown fallback name → hard boot error; valid name with unset creds
  → skip-with-warning (fail-open, lets ops stage the flag ahead of the
  SES_* secret). If all fallbacks skip, boots with the primary only.
- config: EMAIL_PROVIDER_FALLBACK comma-separated → ordered Fallbacks.
- metric: instant_email_failover_total{outcome=primary_ok|fallback_ok|
  all_failed}. INFO on fallback engage (work performed), DEBUG on
  primary-ok (idle), ERROR on all_failed. Emails masked in all logs.
  Lazy *Vec — absent from /metrics until first emit (inert default).
- tests: in-package failover_provider_test.go (fake recording provider):
  primary-ok-no-fallback-call, primary-fail→fallback-ok (transient/
  permanent/skip), all-fail→last-error, all-skip→no all_failed metric,
  ordering, Name(), empty/nil guards, per-outcome metric increments,
  email-masking on every log path. Factory inert-default + with-fallback
  + unknown-name-errors + missing-creds-skipped. Config parse table.

internal/email per-package coverage 96.4% (≥95 floor); all new funcs
100%. make gate green. Infra alert/tile/catalog ship in a separate
infra PR (rule 25) — referenced in the PR body.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: mastermanas805

internal/config/config.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"strings"
 )
 
 // Config holds all runtime configuration for the worker service.
@@ -53,6 +54,16 @@ type Config struct {
 	SESFromEmail     string            // SES_FROM_EMAIL (must be a verified SES identity)
 	SESTemplateNames map[string]string // SES_TEMPLATE_NAMES (JSON object: audit_log.kind → SES template name)
 
+	// EmailProviderFallback is the EMAIL_PROVIDER_FALLBACK env var: a
+	// comma-separated, ordered list of secondary provider names (e.g. "ses")
+	// the failover provider retries through when the primary errors or hard-
+	// rejects a send. Inert by default — unset/empty means single-provider
+	// mode, byte-identical to the pre-failover worker. Each named fallback
+	// reuses that provider's existing sub-config (Brevo*/SES*). A fallback
+	// whose creds are unset is skipped-with-warning at boot (not fatal) so an
+	// operator can stage EMAIL_PROVIDER_FALLBACK ahead of the secret.
+	EmailProviderFallback []string // EMAIL_PROVIDER_FALLBACK (comma-separated)
+
 	Environment       string // ENVIRONMENT
 	MaxMindLicenseKey string // MAXMIND_LICENSE_KEY — for GeoLite2 refresh job
 	GeoLite2DBPath    string // GEOLITE2_DB_PATH — local path to the GeoLite2 City MMDB
@@ -196,30 +207,31 @@ func require(key string) string {
 // Load reads configuration from environment variables. Panics on missing required fields.
 func Load() *Config {
 	cfg := &Config{
-		DatabaseURL:          require("DATABASE_URL"),
-		RedisURL:             getenv("REDIS_URL", "redis://localhost:6379"),
-		ProvisionerAddr:      os.Getenv("PROVISIONER_ADDR"),
-		ProvisionerSecret:    os.Getenv("PROVISIONER_SECRET"),
-		EmailProvider:        os.Getenv("EMAIL_PROVIDER"),
-		BrevoAPIKey:          os.Getenv("BREVO_API_KEY"),
-		BrevoTemplateIDs:     parseBrevoTemplateIDs(os.Getenv("BREVO_TEMPLATE_IDS")),
-		BrevoSenderEmail:     os.Getenv("BREVO_SENDER_EMAIL"),
-		BrevoSenderName:      os.Getenv("BREVO_SENDER_NAME"),
-		SESAWSRegion:         os.Getenv("SES_AWS_REGION"),
-		SESAWSAccessKey:      os.Getenv("SES_AWS_ACCESS_KEY_ID"),
-		SESAWSSecretKey:      os.Getenv("SES_AWS_SECRET_ACCESS_KEY"),
-		SESFromEmail:         os.Getenv("SES_FROM_EMAIL"),
-		SESTemplateNames:     parseSESTemplateNames(os.Getenv("SES_TEMPLATE_NAMES")),
-		Environment:          getenv("ENVIRONMENT", "development"),
-		MaxMindLicenseKey:    os.Getenv("MAXMIND_LICENSE_KEY"),
-		GeoLite2DBPath:       getenv("GEOLITE2_DB_PATH", "./GeoLite2-City.mmdb"),
-		PlansPath:            os.Getenv("PLANS_PATH"),
-		ObjectStoreEndpoint:  os.Getenv("OBJECT_STORE_ENDPOINT"),
-		ObjectStoreAccessKey: os.Getenv("OBJECT_STORE_ACCESS_KEY"),
-		ObjectStoreSecretKey: os.Getenv("OBJECT_STORE_SECRET_KEY"),
-		ObjectStoreBucket:    getenv("OBJECT_STORE_BUCKET", "instant-shared"),
-		ObjectStoreRegion:    os.Getenv("OBJECT_STORE_REGION"),
-		ObjectStoreSecure:    os.Getenv("OBJECT_STORE_SECURE") == "true",
+		DatabaseURL:           require("DATABASE_URL"),
+		RedisURL:              getenv("REDIS_URL", "redis://localhost:6379"),
+		ProvisionerAddr:       os.Getenv("PROVISIONER_ADDR"),
+		ProvisionerSecret:     os.Getenv("PROVISIONER_SECRET"),
+		EmailProvider:         os.Getenv("EMAIL_PROVIDER"),
+		BrevoAPIKey:           os.Getenv("BREVO_API_KEY"),
+		BrevoTemplateIDs:      parseBrevoTemplateIDs(os.Getenv("BREVO_TEMPLATE_IDS")),
+		BrevoSenderEmail:      os.Getenv("BREVO_SENDER_EMAIL"),
+		BrevoSenderName:       os.Getenv("BREVO_SENDER_NAME"),
+		SESAWSRegion:          os.Getenv("SES_AWS_REGION"),
+		SESAWSAccessKey:       os.Getenv("SES_AWS_ACCESS_KEY_ID"),
+		SESAWSSecretKey:       os.Getenv("SES_AWS_SECRET_ACCESS_KEY"),
+		SESFromEmail:          os.Getenv("SES_FROM_EMAIL"),
+		SESTemplateNames:      parseSESTemplateNames(os.Getenv("SES_TEMPLATE_NAMES")),
+		EmailProviderFallback: parseCSVProviders(os.Getenv("EMAIL_PROVIDER_FALLBACK")),
+		Environment:           getenv("ENVIRONMENT", "development"),
+		MaxMindLicenseKey:     os.Getenv("MAXMIND_LICENSE_KEY"),
+		GeoLite2DBPath:        getenv("GEOLITE2_DB_PATH", "./GeoLite2-City.mmdb"),
+		PlansPath:             os.Getenv("PLANS_PATH"),
+		ObjectStoreEndpoint:   os.Getenv("OBJECT_STORE_ENDPOINT"),
+		ObjectStoreAccessKey:  os.Getenv("OBJECT_STORE_ACCESS_KEY"),
+		ObjectStoreSecretKey:  os.Getenv("OBJECT_STORE_SECRET_KEY"),
+		ObjectStoreBucket:     getenv("OBJECT_STORE_BUCKET", "instant-shared"),
+		ObjectStoreRegion:     os.Getenv("OBJECT_STORE_REGION"),
+		ObjectStoreSecure:     os.Getenv("OBJECT_STORE_SECURE") == "true",
 
 		MinioEndpoint:     os.Getenv("MINIO_ENDPOINT"),
 		MinioRootUser:     os.Getenv("MINIO_ROOT_USER"),
@@ -372,3 +384,24 @@ func parseSESTemplateNames(raw string) map[string]string {
 	}
 	return m
 }
+
+// parseCSVProviders splits the EMAIL_PROVIDER_FALLBACK env var into an ordered,
+// trimmed list of provider names, dropping empty segments. "" → nil (inert
+// default: no failover). "ses" → ["ses"]. " ses , brevo " → ["ses","brevo"].
+// Returns nil (not an empty slice) when there are no usable entries so the
+// factory's len(cfg.Fallbacks)==0 inert-default check is unambiguous.
+func parseCSVProviders(raw string) []string {
+	if strings.TrimSpace(raw) == "" {
+		return nil
+	}
+	out := make([]string, 0, 2)
+	for _, part := range strings.Split(raw, ",") {
+		if p := strings.TrimSpace(part); p != "" {
+			out = append(out, p)
+		}
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}

internal/config/config_test.go

@@ -306,3 +306,56 @@ func TestParseSESTemplateNames(t *testing.T) {
 		t.Errorf("malformed should be empty map: %v", bad)
 	}
 }
+
+func TestParseCSVProviders(t *testing.T) {
+	cases := []struct {
+		name string
+		raw  string
+		want []string
+	}{
+		{"empty", "", nil},
+		{"whitespace-only", "   ", nil},
+		{"single", "ses", []string{"ses"}},
+		{"ordered-pair", "ses,brevo", []string{"ses", "brevo"}},
+		{"trims-and-drops-blanks", " ses , , brevo ", []string{"ses", "brevo"}},
+		{"trailing-comma", "ses,", []string{"ses"}},
+		{"leading-comma", ",ses", []string{"ses"}},
+		{"all-blank-segments", " , , ", nil},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := parseCSVProviders(tc.raw)
+			if len(got) != len(tc.want) {
+				t.Fatalf("parseCSVProviders(%q) = %v; want %v", tc.raw, got, tc.want)
+			}
+			for i := range got {
+				if got[i] != tc.want[i] {
+					t.Errorf("parseCSVProviders(%q)[%d] = %q; want %q", tc.raw, i, got[i], tc.want[i])
+				}
+			}
+		})
+	}
+}
+
+// TestLoad_EmailProviderFallback — the env var flows through Load() into the
+// ordered Fallbacks slice; unset → nil (inert default).
+func TestLoad_EmailProviderFallback(t *testing.T) {
+	t.Setenv("DATABASE_URL", "postgres://x")
+	t.Setenv("EMAIL_PROVIDER_FALLBACK", "ses, brevo")
+	cfg := Load()
+	if len(cfg.EmailProviderFallback) != 2 ||
+		cfg.EmailProviderFallback[0] != "ses" || cfg.EmailProviderFallback[1] != "brevo" {
+		t.Errorf("EmailProviderFallback = %v; want [ses brevo]", cfg.EmailProviderFallback)
+	}
+}
+
+// TestLoad_EmailProviderFallback_UnsetIsNil — the inert default at the config
+// layer: no env var → nil slice → single-provider mode.
+func TestLoad_EmailProviderFallback_UnsetIsNil(t *testing.T) {
+	t.Setenv("DATABASE_URL", "postgres://x")
+	t.Setenv("EMAIL_PROVIDER_FALLBACK", "")
+	cfg := Load()
+	if cfg.EmailProviderFallback != nil {
+		t.Errorf("EmailProviderFallback = %v; want nil (inert default)", cfg.EmailProviderFallback)
+	}
+}

internal/email/factory.go

@@ -11,6 +11,7 @@ package email
 import (
 	"fmt"
 	"log/slog"
+	"strings"
 )
 
 // Provider identifier constants — used both by env-var parsing in
@@ -48,14 +49,101 @@ type Config struct {
 
 	// Future: SendGrid SendGridConfig. Keep grouped so adding a
 	// provider doesn't pollute the top-level Config namespace.
+
+	// Fallbacks is the ordered EMAIL_PROVIDER_FALLBACK list (comma-separated
+	// provider names, e.g. ["ses"]). When non-empty, NewProvider builds the
+	// primary (cfg.Provider) plus each named fallback via the SAME switch and
+	// returns a FailoverProvider that tries them in order. When empty/unset
+	// the behavior is byte-identical to single-provider mode — this is the
+	// inert-by-default safety contract (root CLAUDE.md flag-protect rule):
+	// with no secondary configured the worker returns the bare single
+	// provider exactly as it did before this feature existed.
+	//
+	// Each fallback uses the same per-provider sub-config already on Config
+	// (Brevo/SES). An operator wiring SES as a fallback sets EMAIL_PROVIDER=
+	// brevo, EMAIL_PROVIDER_FALLBACK=ses, and the SES_* env vars; both share
+	// this one Config.
+	Fallbacks []string
 }
 
-// NewProvider builds the EmailProvider selected by cfg.Provider. Returns
-// an error only when the provider name is set to an unknown value — empty
-// or "noop" deliberately succeeds with NoopProvider so a missing env var
-// doesn't crash the worker at boot.
+// NewProvider builds the EmailProvider selected by cfg.Provider, optionally
+// wrapped in a FailoverProvider when cfg.Fallbacks is non-empty.
+//
+// Returns an error only when a provider name (primary OR a fallback) is set to
+// an unknown value — empty or "noop" for the PRIMARY deliberately succeeds
+// with NoopProvider so a missing env var doesn't crash the worker at boot.
+//
+// Inert default: cfg.Fallbacks empty/unset → returns the bare single provider
+// exactly as before (no FailoverProvider wrapper). This is the safety contract;
+// TestFactory_NoFallback_ReturnsBareProvider pins it.
+//
+// Fallback construction is fail-OPEN on missing creds: a fallback whose
+// provider name is valid but whose required creds are unset (e.g. SES_* not
+// yet wired) is logged with a clear warning and SKIPPED, not fatal. This lets
+// an operator set EMAIL_PROVIDER_FALLBACK=ses ahead of the SES_* secrets
+// landing without wedging the worker boot. An UNKNOWN fallback name is still a
+// hard error (same strictness as the primary switch) — that's a config typo,
+// not a not-yet-provisioned credential. If every fallback skips, the worker
+// boots with just the primary (no wrapper), preserving today's behavior.
 func NewProvider(cfg Config) (EmailProvider, error) {
-	switch cfg.Provider {
+	primary, err := buildOne(cfg.Provider, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	// Inert default — no fallbacks configured → single provider, byte-identical
+	// to the pre-failover behavior. No FailoverProvider, no metric, no wrapper.
+	if len(cfg.Fallbacks) == 0 {
+		return primary, nil
+	}
+
+	chain := []EmailProvider{primary}
+	for _, name := range cfg.Fallbacks {
+		name = strings.TrimSpace(name)
+		if name == "" {
+			continue // tolerate "ses," / " , ses" sloppiness in the env var
+		}
+		fb, ferr := buildOne(name, cfg)
+		if ferr != nil {
+			if isMissingCredsErr(name, ferr) {
+				// Fail-open: valid provider, creds not yet wired. Skip it so
+				// the operator can stage EMAIL_PROVIDER_FALLBACK ahead of the
+				// secret. Warn loudly so the gap is visible.
+				slog.Warn("email.failover.fallback_skipped",
+					"fallback", name,
+					"reason", "required credentials unset — fallback unavailable until configured",
+					"error", ferr,
+				)
+				continue
+			}
+			// Unknown provider name (config typo) → hard boot error.
+			return nil, fmt.Errorf("email: invalid EMAIL_PROVIDER_FALLBACK %q: %w", name, ferr)
+		}
+		chain = append(chain, fb)
+	}
+
+	// Every fallback skipped (creds unset) → fall back to the bare primary so
+	// the worker still boots and sends via the primary. No wrapper, same as
+	// the inert default.
+	if len(chain) == 1 {
+		slog.Warn("email.failover.no_usable_fallbacks",
+			"reason", "EMAIL_PROVIDER_FALLBACK set but every fallback was skipped (creds unset) — using primary only",
+			"primary", primary.Name(),
+		)
+		return primary, nil
+	}
+
+	return NewFailoverProvider(chain...)
+}
+
+// buildOne constructs a single concrete EmailProvider by name, sharing the
+// per-provider sub-configs on cfg. This is the ONE switch used by both the
+// primary and every fallback — adding a provider means one new case here and
+// it is instantly available as both a primary and a fallback (no duplicated
+// switch). An empty/"noop" name yields NoopProvider (fail-open default for the
+// primary); used as a fallback it is a harmless no-op that always skips.
+func buildOne(name string, cfg Config) (EmailProvider, error) {
+	switch name {
 	case "", providerNameNoop:
 		// Fail-open: an operator who hasn't configured an email provider
 		// gets silent no-ops, not a boot crash. The warning surfaces
@@ -75,7 +163,24 @@ func NewProvider(cfg Config) (EmailProvider, error) {
 	//     return NewSendGridProvider(cfg.SendGrid)
 
 	default:
-		return nil, fmt.Errorf("email: unknown EMAIL_PROVIDER %q (supported: %q, %q, %q, %q)",
-			cfg.Provider, providerNameNoop, providerNameBrevo, providerNameSES, "")
+		return nil, fmt.Errorf("email: unknown provider %q (supported: %q, %q, %q, %q)",
+			name, providerNameNoop, providerNameBrevo, providerNameSES, "")
+	}
+}
+
+// isMissingCredsErr reports whether a fallback construction error is a
+// "required credential unset" condition (skip-with-warning) rather than an
+// unknown-name typo (hard error). The provider constructors return a known
+// non-nil error in exactly the missing-creds case; an unknown name is caught
+// by the buildOne default branch and its message begins "email: unknown
+// provider". We classify by "is the name a real provider" — if buildOne knew
+// the name, any error from it is a construction/creds error, which we treat as
+// skip-with-warning. This keeps the policy decision in one place.
+func isMissingCredsErr(name string, _ error) bool {
+	switch name {
+	case providerNameBrevo, providerNameSES:
+		return true // known provider whose constructor failed → missing creds
+	default:
+		return false // unknown name → hard error
 	}
 }