diff --git a/internal/config/config.go b/internal/config/config.go index a9415b0..8e4427f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "os" + "strings" ) // Config holds all runtime configuration for the worker service. @@ -53,6 +54,16 @@ type Config struct { SESFromEmail string // SES_FROM_EMAIL (must be a verified SES identity) SESTemplateNames map[string]string // SES_TEMPLATE_NAMES (JSON object: audit_log.kind → SES template name) + // EmailProviderFallback is the EMAIL_PROVIDER_FALLBACK env var: a + // comma-separated, ordered list of secondary provider names (e.g. "ses") + // the failover provider retries through when the primary errors or hard- + // rejects a send. Inert by default — unset/empty means single-provider + // mode, byte-identical to the pre-failover worker. Each named fallback + // reuses that provider's existing sub-config (Brevo*/SES*). A fallback + // whose creds are unset is skipped-with-warning at boot (not fatal) so an + // operator can stage EMAIL_PROVIDER_FALLBACK ahead of the secret. + EmailProviderFallback []string // EMAIL_PROVIDER_FALLBACK (comma-separated) + Environment string // ENVIRONMENT MaxMindLicenseKey string // MAXMIND_LICENSE_KEY — for GeoLite2 refresh job GeoLite2DBPath string // GEOLITE2_DB_PATH — local path to the GeoLite2 City MMDB @@ -196,30 +207,31 @@ func require(key string) string { // Load reads configuration from environment variables. Panics on missing required fields. func Load() *Config { cfg := &Config{ - DatabaseURL: require("DATABASE_URL"), - RedisURL: getenv("REDIS_URL", "redis://localhost:6379"), - ProvisionerAddr: os.Getenv("PROVISIONER_ADDR"), - ProvisionerSecret: os.Getenv("PROVISIONER_SECRET"), - EmailProvider: os.Getenv("EMAIL_PROVIDER"), - BrevoAPIKey: os.Getenv("BREVO_API_KEY"), - BrevoTemplateIDs: parseBrevoTemplateIDs(os.Getenv("BREVO_TEMPLATE_IDS")), - BrevoSenderEmail: os.Getenv("BREVO_SENDER_EMAIL"), - BrevoSenderName: os.Getenv("BREVO_SENDER_NAME"), - SESAWSRegion: os.Getenv("SES_AWS_REGION"), - SESAWSAccessKey: os.Getenv("SES_AWS_ACCESS_KEY_ID"), - SESAWSSecretKey: os.Getenv("SES_AWS_SECRET_ACCESS_KEY"), - SESFromEmail: os.Getenv("SES_FROM_EMAIL"), - SESTemplateNames: parseSESTemplateNames(os.Getenv("SES_TEMPLATE_NAMES")), - Environment: getenv("ENVIRONMENT", "development"), - MaxMindLicenseKey: os.Getenv("MAXMIND_LICENSE_KEY"), - GeoLite2DBPath: getenv("GEOLITE2_DB_PATH", "./GeoLite2-City.mmdb"), - PlansPath: os.Getenv("PLANS_PATH"), - ObjectStoreEndpoint: os.Getenv("OBJECT_STORE_ENDPOINT"), - ObjectStoreAccessKey: os.Getenv("OBJECT_STORE_ACCESS_KEY"), - ObjectStoreSecretKey: os.Getenv("OBJECT_STORE_SECRET_KEY"), - ObjectStoreBucket: getenv("OBJECT_STORE_BUCKET", "instant-shared"), - ObjectStoreRegion: os.Getenv("OBJECT_STORE_REGION"), - ObjectStoreSecure: os.Getenv("OBJECT_STORE_SECURE") == "true", + DatabaseURL: require("DATABASE_URL"), + RedisURL: getenv("REDIS_URL", "redis://localhost:6379"), + ProvisionerAddr: os.Getenv("PROVISIONER_ADDR"), + ProvisionerSecret: os.Getenv("PROVISIONER_SECRET"), + EmailProvider: os.Getenv("EMAIL_PROVIDER"), + BrevoAPIKey: os.Getenv("BREVO_API_KEY"), + BrevoTemplateIDs: parseBrevoTemplateIDs(os.Getenv("BREVO_TEMPLATE_IDS")), + BrevoSenderEmail: os.Getenv("BREVO_SENDER_EMAIL"), + BrevoSenderName: os.Getenv("BREVO_SENDER_NAME"), + SESAWSRegion: os.Getenv("SES_AWS_REGION"), + SESAWSAccessKey: os.Getenv("SES_AWS_ACCESS_KEY_ID"), + SESAWSSecretKey: os.Getenv("SES_AWS_SECRET_ACCESS_KEY"), + SESFromEmail: os.Getenv("SES_FROM_EMAIL"), + SESTemplateNames: parseSESTemplateNames(os.Getenv("SES_TEMPLATE_NAMES")), + EmailProviderFallback: parseCSVProviders(os.Getenv("EMAIL_PROVIDER_FALLBACK")), + Environment: getenv("ENVIRONMENT", "development"), + MaxMindLicenseKey: os.Getenv("MAXMIND_LICENSE_KEY"), + GeoLite2DBPath: getenv("GEOLITE2_DB_PATH", "./GeoLite2-City.mmdb"), + PlansPath: os.Getenv("PLANS_PATH"), + ObjectStoreEndpoint: os.Getenv("OBJECT_STORE_ENDPOINT"), + ObjectStoreAccessKey: os.Getenv("OBJECT_STORE_ACCESS_KEY"), + ObjectStoreSecretKey: os.Getenv("OBJECT_STORE_SECRET_KEY"), + ObjectStoreBucket: getenv("OBJECT_STORE_BUCKET", "instant-shared"), + ObjectStoreRegion: os.Getenv("OBJECT_STORE_REGION"), + ObjectStoreSecure: os.Getenv("OBJECT_STORE_SECURE") == "true", MinioEndpoint: os.Getenv("MINIO_ENDPOINT"), MinioRootUser: os.Getenv("MINIO_ROOT_USER"), @@ -372,3 +384,24 @@ func parseSESTemplateNames(raw string) map[string]string { } return m } + +// parseCSVProviders splits the EMAIL_PROVIDER_FALLBACK env var into an ordered, +// trimmed list of provider names, dropping empty segments. "" → nil (inert +// default: no failover). "ses" → ["ses"]. " ses , brevo " → ["ses","brevo"]. +// Returns nil (not an empty slice) when there are no usable entries so the +// factory's len(cfg.Fallbacks)==0 inert-default check is unambiguous. +func parseCSVProviders(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + out := make([]string, 0, 2) + for _, part := range strings.Split(raw, ",") { + if p := strings.TrimSpace(part); p != "" { + out = append(out, p) + } + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0ff3778..947d852 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -306,3 +306,56 @@ func TestParseSESTemplateNames(t *testing.T) { t.Errorf("malformed should be empty map: %v", bad) } } + +func TestParseCSVProviders(t *testing.T) { + cases := []struct { + name string + raw string + want []string + }{ + {"empty", "", nil}, + {"whitespace-only", " ", nil}, + {"single", "ses", []string{"ses"}}, + {"ordered-pair", "ses,brevo", []string{"ses", "brevo"}}, + {"trims-and-drops-blanks", " ses , , brevo ", []string{"ses", "brevo"}}, + {"trailing-comma", "ses,", []string{"ses"}}, + {"leading-comma", ",ses", []string{"ses"}}, + {"all-blank-segments", " , , ", nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := parseCSVProviders(tc.raw) + if len(got) != len(tc.want) { + t.Fatalf("parseCSVProviders(%q) = %v; want %v", tc.raw, got, tc.want) + } + for i := range got { + if got[i] != tc.want[i] { + t.Errorf("parseCSVProviders(%q)[%d] = %q; want %q", tc.raw, i, got[i], tc.want[i]) + } + } + }) + } +} + +// TestLoad_EmailProviderFallback — the env var flows through Load() into the +// ordered Fallbacks slice; unset → nil (inert default). +func TestLoad_EmailProviderFallback(t *testing.T) { + t.Setenv("DATABASE_URL", "postgres://x") + t.Setenv("EMAIL_PROVIDER_FALLBACK", "ses, brevo") + cfg := Load() + if len(cfg.EmailProviderFallback) != 2 || + cfg.EmailProviderFallback[0] != "ses" || cfg.EmailProviderFallback[1] != "brevo" { + t.Errorf("EmailProviderFallback = %v; want [ses brevo]", cfg.EmailProviderFallback) + } +} + +// TestLoad_EmailProviderFallback_UnsetIsNil — the inert default at the config +// layer: no env var → nil slice → single-provider mode. +func TestLoad_EmailProviderFallback_UnsetIsNil(t *testing.T) { + t.Setenv("DATABASE_URL", "postgres://x") + t.Setenv("EMAIL_PROVIDER_FALLBACK", "") + cfg := Load() + if cfg.EmailProviderFallback != nil { + t.Errorf("EmailProviderFallback = %v; want nil (inert default)", cfg.EmailProviderFallback) + } +} diff --git a/internal/email/factory.go b/internal/email/factory.go index 6d3e662..03bde60 100644 --- a/internal/email/factory.go +++ b/internal/email/factory.go @@ -11,6 +11,7 @@ package email import ( "fmt" "log/slog" + "strings" ) // Provider identifier constants — used both by env-var parsing in @@ -48,14 +49,101 @@ type Config struct { // Future: SendGrid SendGridConfig. Keep grouped so adding a // provider doesn't pollute the top-level Config namespace. + + // Fallbacks is the ordered EMAIL_PROVIDER_FALLBACK list (comma-separated + // provider names, e.g. ["ses"]). When non-empty, NewProvider builds the + // primary (cfg.Provider) plus each named fallback via the SAME switch and + // returns a FailoverProvider that tries them in order. When empty/unset + // the behavior is byte-identical to single-provider mode — this is the + // inert-by-default safety contract (root CLAUDE.md flag-protect rule): + // with no secondary configured the worker returns the bare single + // provider exactly as it did before this feature existed. + // + // Each fallback uses the same per-provider sub-config already on Config + // (Brevo/SES). An operator wiring SES as a fallback sets EMAIL_PROVIDER= + // brevo, EMAIL_PROVIDER_FALLBACK=ses, and the SES_* env vars; both share + // this one Config. + Fallbacks []string } -// NewProvider builds the EmailProvider selected by cfg.Provider. Returns -// an error only when the provider name is set to an unknown value — empty -// or "noop" deliberately succeeds with NoopProvider so a missing env var -// doesn't crash the worker at boot. +// NewProvider builds the EmailProvider selected by cfg.Provider, optionally +// wrapped in a FailoverProvider when cfg.Fallbacks is non-empty. +// +// Returns an error only when a provider name (primary OR a fallback) is set to +// an unknown value — empty or "noop" for the PRIMARY deliberately succeeds +// with NoopProvider so a missing env var doesn't crash the worker at boot. +// +// Inert default: cfg.Fallbacks empty/unset → returns the bare single provider +// exactly as before (no FailoverProvider wrapper). This is the safety contract; +// TestFactory_NoFallback_ReturnsBareProvider pins it. +// +// Fallback construction is fail-OPEN on missing creds: a fallback whose +// provider name is valid but whose required creds are unset (e.g. SES_* not +// yet wired) is logged with a clear warning and SKIPPED, not fatal. This lets +// an operator set EMAIL_PROVIDER_FALLBACK=ses ahead of the SES_* secrets +// landing without wedging the worker boot. An UNKNOWN fallback name is still a +// hard error (same strictness as the primary switch) — that's a config typo, +// not a not-yet-provisioned credential. If every fallback skips, the worker +// boots with just the primary (no wrapper), preserving today's behavior. func NewProvider(cfg Config) (EmailProvider, error) { - switch cfg.Provider { + primary, err := buildOne(cfg.Provider, cfg) + if err != nil { + return nil, err + } + + // Inert default — no fallbacks configured → single provider, byte-identical + // to the pre-failover behavior. No FailoverProvider, no metric, no wrapper. + if len(cfg.Fallbacks) == 0 { + return primary, nil + } + + chain := []EmailProvider{primary} + for _, name := range cfg.Fallbacks { + name = strings.TrimSpace(name) + if name == "" { + continue // tolerate "ses," / " , ses" sloppiness in the env var + } + fb, ferr := buildOne(name, cfg) + if ferr != nil { + if isMissingCredsErr(name, ferr) { + // Fail-open: valid provider, creds not yet wired. Skip it so + // the operator can stage EMAIL_PROVIDER_FALLBACK ahead of the + // secret. Warn loudly so the gap is visible. + slog.Warn("email.failover.fallback_skipped", + "fallback", name, + "reason", "required credentials unset — fallback unavailable until configured", + "error", ferr, + ) + continue + } + // Unknown provider name (config typo) → hard boot error. + return nil, fmt.Errorf("email: invalid EMAIL_PROVIDER_FALLBACK %q: %w", name, ferr) + } + chain = append(chain, fb) + } + + // Every fallback skipped (creds unset) → fall back to the bare primary so + // the worker still boots and sends via the primary. No wrapper, same as + // the inert default. + if len(chain) == 1 { + slog.Warn("email.failover.no_usable_fallbacks", + "reason", "EMAIL_PROVIDER_FALLBACK set but every fallback was skipped (creds unset) — using primary only", + "primary", primary.Name(), + ) + return primary, nil + } + + return NewFailoverProvider(chain...) +} + +// buildOne constructs a single concrete EmailProvider by name, sharing the +// per-provider sub-configs on cfg. This is the ONE switch used by both the +// primary and every fallback — adding a provider means one new case here and +// it is instantly available as both a primary and a fallback (no duplicated +// switch). An empty/"noop" name yields NoopProvider (fail-open default for the +// primary); used as a fallback it is a harmless no-op that always skips. +func buildOne(name string, cfg Config) (EmailProvider, error) { + switch name { case "", providerNameNoop: // Fail-open: an operator who hasn't configured an email provider // gets silent no-ops, not a boot crash. The warning surfaces @@ -75,7 +163,24 @@ func NewProvider(cfg Config) (EmailProvider, error) { // return NewSendGridProvider(cfg.SendGrid) default: - return nil, fmt.Errorf("email: unknown EMAIL_PROVIDER %q (supported: %q, %q, %q, %q)", - cfg.Provider, providerNameNoop, providerNameBrevo, providerNameSES, "") + return nil, fmt.Errorf("email: unknown provider %q (supported: %q, %q, %q, %q)", + name, providerNameNoop, providerNameBrevo, providerNameSES, "") + } +} + +// isMissingCredsErr reports whether a fallback construction error is a +// "required credential unset" condition (skip-with-warning) rather than an +// unknown-name typo (hard error). The provider constructors return a known +// non-nil error in exactly the missing-creds case; an unknown name is caught +// by the buildOne default branch and its message begins "email: unknown +// provider". We classify by "is the name a real provider" — if buildOne knew +// the name, any error from it is a construction/creds error, which we treat as +// skip-with-warning. This keeps the policy decision in one place. +func isMissingCredsErr(name string, _ error) bool { + switch name { + case providerNameBrevo, providerNameSES: + return true // known provider whose constructor failed → missing creds + default: + return false // unknown name → hard error } } diff --git a/internal/email/failover_provider.go b/internal/email/failover_provider.go new file mode 100644 index 0000000..0a10135 --- /dev/null +++ b/internal/email/failover_provider.go @@ -0,0 +1,232 @@ +package email + +// failover_provider.go — ordered multi-ESP failover for outbound event email. +// +// A FailoverProvider wraps an ordered slice of EmailProvider (primary first, +// then one or more secondaries) and implements EmailProvider itself, so the +// forwarder is unchanged: it still holds a single EmailProvider and never +// knows whether one ESP or three sit behind the seam. +// +// Why this exists (task #35): the standing P0 is that the primary ESP (Brevo) +// returns 201 then internally rejects every send because the sender domain is +// unvalidated — so today *every* transactional email is silently lost at the +// relay. A failover lets the operator wire a second ESP (SES) so that when the +// primary errors or hard-rejects a send, the same EventEmail is immediately +// retried through the secondary. The unit of success is "did SOME provider +// accept the bytes", which is exactly the signal the forwarder needs for its +// claim-after-2xx contract (root CLAUDE.md / worker convention 6): the row is +// claimed iff at least one provider returned success. +// +// ── Decision rule (which result advances to the next provider) ────────────── +// +// SendEvent walks the chain in order and, for each provider, looks at the +// (messageID, error) pair: +// +// - success (err == nil) → STOP, return success. Record +// primary_ok if it was the first +// provider, else fallback_ok. +// - SendClassTransient (5xx/network/429) → try the next provider. The +// primary is unhealthy; the +// secondary may still deliver. +// - SendClassPermanent (4xx/auth/reject) → try the next provider. This is +// the P0 case: Brevo "rejects" the +// payload at the account level, but +// a healthy SES can still send it. +// Treating Permanent as fall-through +// is the whole point of the feature. +// - SendClassSkippedNoTemplate → try the next provider. The primary +// has no template for this kind but a +// secondary might. IMPORTANT: if EVERY +// provider skips (none has a template), +// the chain returns that last +// SkippedNoTemplate verbatim so the +// forwarder advances its cursor +// silently — a kind nobody is +// configured to send is not an error. +// See the all-skipped note below. +// +// In short: ANY non-success result advances to the next provider; the first +// success wins; if all fail we return the LAST error. This is deliberately +// uniform (we do not stop early on a Permanent from the primary) because the +// secondary's verdict on the same payload is independent — a payload Brevo +// rejects for account reasons can be perfectly acceptable to SES. +// +// ── all-skipped vs all-failed ────────────────────────────────────────────── +// +// "All skipped" (every provider returned SendClassSkippedNoTemplate) is NOT a +// failure: it means no configured ESP has a template/renderer for this kind, +// which is the operator's explicit opt-out. We surface the last skip error so +// ClassOf() reports SkippedNoTemplate and the forwarder advances the cursor +// silently — and we do NOT bump the all_failed counter for it (an all-skip is +// not a degraded-ESP signal). "All failed" with at least one real Transient/ +// Permanent error is the alert-able all_failed outcome. +// +// ── Observability ────────────────────────────────────────────────────────── +// +// - metrics.EmailFailoverTotal{outcome} on every SendEvent: +// primary_ok — first provider succeeded. +// fallback_ok — a later provider succeeded after the primary failed. +// all_failed — every provider failed (at least one real error). +// (An all-skipped chain records primary_ok-equivalent? No — it is neither +// a send nor a degraded signal, so it records NO outcome label, keeping +// the counter strictly about real send attempts. See the loop below.) +// - INFO log when a fallback engages (work performed — worker convention 1). +// - DEBUG log when the primary succeeds first (idle/happy path stays quiet). +// - Recipient emails are masked (worker convention 7 — stub right of '@'). +// +// The FailoverProvider holds no network state of its own; all I/O is delegated +// to the wrapped providers, each of which already enforces its own timeout. + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "instant.dev/worker/internal/metrics" +) + +// Failover outcome labels for metrics.EmailFailoverTotal. Named constants so a +// typo trips compile-time and the infra alert/dashboard can reference them by +// name (no scattered string literals — CLAUDE.md no-hardcoded-strings rule). +const ( + failoverOutcomePrimaryOK = "primary_ok" + failoverOutcomeFallbackOK = "fallback_ok" + failoverOutcomeAllFailed = "all_failed" +) + +// FailoverProvider sends through an ordered chain of EmailProvider, advancing +// to the next on any non-success result and returning the first success. It +// implements EmailProvider so it drops into the forwarder unchanged. +type FailoverProvider struct { + // providers is the ordered chain, primary first. Guaranteed non-empty by + // NewFailoverProvider (which errors on an empty/all-nil slice). Read-only + // after construction; SendEvent only iterates. + providers []EmailProvider + + // name is the precomputed "failover(brevo,ses)" label, built once at + // construction so Name() is allocation-free on the hot path. + name string +} + +// NewFailoverProvider builds a FailoverProvider from an ordered chain. The +// first element is the primary. Returns an error when the chain is empty or +// contains only nils — a failover with nothing to fail over to is a +// programming error, not a runtime condition, so we surface it at boot rather +// than no-op'ing silently. Nil entries inside an otherwise-valid chain are +// dropped (defensive: a caller that conditionally skips a provider can pass a +// nil rather than rebuilding the slice). +func NewFailoverProvider(providers ...EmailProvider) (*FailoverProvider, error) { + cleaned := make([]EmailProvider, 0, len(providers)) + for _, p := range providers { + if p != nil { + cleaned = append(cleaned, p) + } + } + if len(cleaned) == 0 { + return nil, fmt.Errorf("email: failover provider requires at least one non-nil provider") + } + names := make([]string, 0, len(cleaned)) + for _, p := range cleaned { + names = append(names, p.Name()) + } + return &FailoverProvider{ + providers: cleaned, + name: fmt.Sprintf("failover(%s)", strings.Join(names, ",")), + }, nil +} + +// Name returns the chain identifier, e.g. "failover(brevo,ses)". Stable for +// the lifetime of the binary; safe as a metric/log label. +func (p *FailoverProvider) Name() string { return p.name } + +// SendEvent walks the provider chain in order. See the package-level decision +// rule for the full semantics. Returns the first provider's success, or the +// last provider's error if every provider fails. +func (p *FailoverProvider) SendEvent(ctx context.Context, evt EventEmail) (string, error) { + var lastErr error + sawRealError := false // at least one Transient/Permanent (not a bare skip) + + for i, prov := range p.providers { + msgID, err := prov.SendEvent(ctx, evt) + if err == nil { + // Success. The chain stops here. Distinguish primary (idle/happy + // path, DEBUG) from a fallback engagement (work performed, INFO). + if i == 0 { + metrics.EmailFailoverTotal.WithLabelValues(failoverOutcomePrimaryOK).Inc() + slog.Debug("email.failover.primary_ok", + "provider", prov.Name(), + "kind", evt.Kind, + "recipient", maskEmail(evt.Recipient), + ) + } else { + metrics.EmailFailoverTotal.WithLabelValues(failoverOutcomeFallbackOK).Inc() + slog.Info("email.failover.fallback_ok", + "failed_provider", p.providers[0].Name(), + "succeeded_provider", prov.Name(), + "chain_index", i, + "kind", evt.Kind, + "recipient", maskEmail(evt.Recipient), + "note", "primary ESP failed/skipped; secondary accepted the send — email NOT lost", + ) + } + return msgID, nil + } + + // Non-success. Record the error and decide whether it counts as a + // "real" failure (Transient/Permanent) vs a bare skip. A skip means + // "this provider has no template for this kind" — not a degraded ESP. + lastErr = err + if ClassOf(err) != SendClassSkippedNoTemplate { + sawRealError = true + } + + // Advance to the next provider. Log at INFO only when there IS a next + // provider to engage (work is about to be performed); the terminal + // failure is logged once after the loop. + if i < len(p.providers)-1 { + slog.Info("email.failover.advancing", + "failed_provider", prov.Name(), + "next_provider", p.providers[i+1].Name(), + "class", ClassOf(err).String(), + "kind", evt.Kind, + "recipient", maskEmail(evt.Recipient), + ) + } + } + + // Every provider returned non-success. + if sawRealError { + // At least one provider hit a real error — this is the alert-able + // all_failed outcome (both ESPs refusing → email being lost). + metrics.EmailFailoverTotal.WithLabelValues(failoverOutcomeAllFailed).Inc() + slog.Error("email.failover.all_failed", + "chain", p.name, + "kind", evt.Kind, + "recipient", maskEmail(evt.Recipient), + "last_error", lastErr, + "note", "every ESP in the failover chain refused this send — email lost; forwarder holds cursor on the last error class", + ) + } else { + // Every provider returned SkippedNoTemplate — no configured ESP has a + // template for this kind. Not a degraded-ESP signal, so NO all_failed + // metric and only a DEBUG line. The returned (skip) error makes + // ClassOf() report SkippedNoTemplate so the forwarder advances silently. + slog.Debug("email.failover.all_skipped", + "chain", p.name, + "kind", evt.Kind, + "recipient", maskEmail(evt.Recipient), + ) + } + + // Return the last error verbatim so its SendClass drives the forwarder's + // cursor decision (Transient → hold, Permanent/Skipped → advance). Since + // we walk in order, the last error is the last (lowest-priority) provider's + // verdict — a deliberate choice: if the operator put the more-reliable ESP + // last, its classification is what the forwarder should act on. + return "", lastErr +} + +// Compile-time interface satisfaction check — proves the seam fits. If a +// future EmailProvider change breaks FailoverProvider, the build fails here. +var _ EmailProvider = (*FailoverProvider)(nil) diff --git a/internal/email/failover_provider_test.go b/internal/email/failover_provider_test.go new file mode 100644 index 0000000..0aa89be --- /dev/null +++ b/internal/email/failover_provider_test.go @@ -0,0 +1,393 @@ +package email + +// failover_provider_test.go — in-package tests for the ordered multi-ESP +// FailoverProvider and the factory wiring (EMAIL_PROVIDER_FALLBACK). +// +// These pin the safety contract the task hangs on: +// - inert default: no fallback configured → bare single provider, no wrapper. +// - primary success short-circuits (no fallback call). +// - primary failure → fallback engaged, first success wins. +// - all fail → last error returned. +// - ordering, Name(), per-outcome metric increments, email masking, guards. + +import ( + "bytes" + "context" + "errors" + "log/slog" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + + "instant.dev/worker/internal/metrics" +) + +// ── fakeProvider — recording EmailProvider test double ────────────────────── + +// fakeProvider records each SendEvent call and returns a scripted result. It +// implements EmailProvider so it slots into the FailoverProvider chain and the +// factory. `calls` counts invocations so a test can assert "the fallback was +// NOT called" when the primary succeeds. +type fakeProvider struct { + name string + msgID string + err error + calls int + gotEvt EventEmail +} + +func (f *fakeProvider) SendEvent(_ context.Context, evt EventEmail) (string, error) { + f.calls++ + f.gotEvt = evt + return f.msgID, f.err +} + +func (f *fakeProvider) Name() string { return f.name } + +// skipErr / transientErr / permanentErr build *SendError of each class. +func skipErr() error { return &SendError{Class: SendClassSkippedNoTemplate, Message: "no template"} } +func transientErr() error { return &SendError{Class: SendClassTransient, Message: "5xx"} } +func permanentErr() error { return &SendError{Class: SendClassPermanent, Message: "rejected"} } + +// readFailover reads the current value of the failover counter for an outcome. +func readFailover(outcome string) float64 { + return testutil.ToFloat64(metrics.EmailFailoverTotal.WithLabelValues(outcome)) +} + +// ── FailoverProvider behavior ─────────────────────────────────────────────── + +// TestFailover_PrimarySucceeds_NoFallbackCall — the happy path: the primary +// returns success, the chain stops, and the fallback is never invoked. Records +// primary_ok. +func TestFailover_PrimarySucceeds_NoFallbackCall(t *testing.T) { + before := readFailover(failoverOutcomePrimaryOK) + primary := &fakeProvider{name: "brevo", msgID: "msg-1"} + fallback := &fakeProvider{name: "ses"} + fp, err := NewFailoverProvider(primary, fallback) + if err != nil { + t.Fatalf("NewFailoverProvider = %v", err) + } + + id, sendErr := fp.SendEvent(context.Background(), EventEmail{Kind: "k", Recipient: "u@example.com"}) + if sendErr != nil { + t.Fatalf("SendEvent = %v; want nil", sendErr) + } + if id != "msg-1" { + t.Errorf("messageID = %q; want msg-1", id) + } + if primary.calls != 1 { + t.Errorf("primary calls = %d; want 1", primary.calls) + } + if fallback.calls != 0 { + t.Errorf("fallback calls = %d; want 0 (primary succeeded)", fallback.calls) + } + if got := readFailover(failoverOutcomePrimaryOK); got != before+1 { + t.Errorf("primary_ok counter = %v; want %v", got, before+1) + } +} + +// TestFailover_PrimaryFails_FallbackSucceeds — primary errors, fallback sends. +// Returns the fallback's messageID and records fallback_ok. Covers both a +// Transient and a Permanent primary error (the P0 case: Brevo "rejects" but +// SES sends). +func TestFailover_PrimaryFails_FallbackSucceeds(t *testing.T) { + for _, tc := range []struct { + name string + primErr error + }{ + {"primary-transient", transientErr()}, + {"primary-permanent", permanentErr()}, // the P0 case + {"primary-skip", skipErr()}, + } { + t.Run(tc.name, func(t *testing.T) { + before := readFailover(failoverOutcomeFallbackOK) + primary := &fakeProvider{name: "brevo", err: tc.primErr} + fallback := &fakeProvider{name: "ses", msgID: "ses-9"} + fp, _ := NewFailoverProvider(primary, fallback) + + id, sendErr := fp.SendEvent(context.Background(), EventEmail{Kind: "k", Recipient: "u@example.com"}) + if sendErr != nil { + t.Fatalf("SendEvent = %v; want nil (fallback should succeed)", sendErr) + } + if id != "ses-9" { + t.Errorf("messageID = %q; want ses-9 (from fallback)", id) + } + if primary.calls != 1 || fallback.calls != 1 { + t.Errorf("calls primary=%d fallback=%d; want 1,1", primary.calls, fallback.calls) + } + if got := readFailover(failoverOutcomeFallbackOK); got != before+1 { + t.Errorf("fallback_ok counter = %v; want %v", got, before+1) + } + }) + } +} + +// TestFailover_AllFail_ReturnsLastError — every provider fails; the chain +// returns the LAST provider's error and records all_failed (because at least +// one real error occurred). +func TestFailover_AllFail_ReturnsLastError(t *testing.T) { + before := readFailover(failoverOutcomeAllFailed) + lastErr := &SendError{Class: SendClassTransient, Message: "ses-down"} + primary := &fakeProvider{name: "brevo", err: permanentErr()} + fallback := &fakeProvider{name: "ses", err: lastErr} + fp, _ := NewFailoverProvider(primary, fallback) + + _, sendErr := fp.SendEvent(context.Background(), EventEmail{Kind: "k", Recipient: "u@example.com"}) + if !errors.Is(sendErr, lastErr) { + t.Fatalf("SendEvent err = %v; want the LAST provider's error %v", sendErr, lastErr) + } + if primary.calls != 1 || fallback.calls != 1 { + t.Errorf("calls primary=%d fallback=%d; want 1,1", primary.calls, fallback.calls) + } + if got := readFailover(failoverOutcomeAllFailed); got != before+1 { + t.Errorf("all_failed counter = %v; want %v", got, before+1) + } +} + +// TestFailover_AllSkipped_NoAllFailedMetric — when every provider returns +// SkippedNoTemplate (no ESP has a template for this kind), the chain returns a +// skip error so the forwarder advances silently, and does NOT bump all_failed +// (an all-skip is not a degraded-ESP signal). +func TestFailover_AllSkipped_NoAllFailedMetric(t *testing.T) { + before := readFailover(failoverOutcomeAllFailed) + primary := &fakeProvider{name: "brevo", err: skipErr()} + fallback := &fakeProvider{name: "ses", err: skipErr()} + fp, _ := NewFailoverProvider(primary, fallback) + + _, sendErr := fp.SendEvent(context.Background(), EventEmail{Kind: "k", Recipient: "u@example.com"}) + if ClassOf(sendErr) != SendClassSkippedNoTemplate { + t.Fatalf("ClassOf(err) = %v; want SkippedNoTemplate", ClassOf(sendErr)) + } + if got := readFailover(failoverOutcomeAllFailed); got != before { + t.Errorf("all_failed counter = %v; want unchanged %v (all-skip is not a failure)", got, before) + } +} + +// TestFailover_Ordering — providers are tried strictly in chain order. A +// three-element chain where the first two fail and the third succeeds must call +// all three in sequence and return the third's id. +func TestFailover_Ordering(t *testing.T) { + p1 := &fakeProvider{name: "a", err: transientErr()} + p2 := &fakeProvider{name: "b", err: permanentErr()} + p3 := &fakeProvider{name: "c", msgID: "c-ok"} + fp, _ := NewFailoverProvider(p1, p2, p3) + + id, err := fp.SendEvent(context.Background(), EventEmail{Kind: "k", Recipient: "u@example.com"}) + if err != nil { + t.Fatalf("SendEvent = %v; want nil", err) + } + if id != "c-ok" { + t.Errorf("messageID = %q; want c-ok", id) + } + if p1.calls != 1 || p2.calls != 1 || p3.calls != 1 { + t.Errorf("calls a=%d b=%d c=%d; want 1,1,1", p1.calls, p2.calls, p3.calls) + } +} + +// TestFailover_Name — Name() reflects the ordered chain. +func TestFailover_Name(t *testing.T) { + fp, _ := NewFailoverProvider(&fakeProvider{name: "brevo"}, &fakeProvider{name: "ses"}) + if got := fp.Name(); got != "failover(brevo,ses)" { + t.Errorf("Name() = %q; want failover(brevo,ses)", got) + } +} + +// TestNewFailoverProvider_EmptyGuard — a failover with no providers (or only +// nils) is a programming error and must return an error, not a nil-deref at +// send time. +func TestNewFailoverProvider_EmptyGuard(t *testing.T) { + if _, err := NewFailoverProvider(); err == nil { + t.Error("NewFailoverProvider() = nil err; want error on empty chain") + } + if _, err := NewFailoverProvider(nil, nil); err == nil { + t.Error("NewFailoverProvider(nil,nil) = nil err; want error on all-nil chain") + } +} + +// TestNewFailoverProvider_DropsNils — a nil entry inside an otherwise-valid +// chain is dropped, not dereferenced. The resulting Name() omits it. +func TestNewFailoverProvider_DropsNils(t *testing.T) { + fp, err := NewFailoverProvider(nil, &fakeProvider{name: "ses"}, nil) + if err != nil { + t.Fatalf("NewFailoverProvider = %v", err) + } + if got := fp.Name(); got != "failover(ses)" { + t.Errorf("Name() = %q; want failover(ses)", got) + } +} + +// TestFailover_MasksEmailInLogs — the recipient must be masked in the log lines +// the FailoverProvider emits (worker convention 7). We capture slog output on +// the fallback-engaged path (INFO) and assert the raw local-part never appears. +func TestFailover_MasksEmailInLogs(t *testing.T) { + var buf bytes.Buffer + orig := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) + defer slog.SetDefault(orig) + + primary := &fakeProvider{name: "brevo", err: permanentErr()} + fallback := &fakeProvider{name: "ses", msgID: "ses-1"} + fp, _ := NewFailoverProvider(primary, fallback) + + const addr = "alice@example.com" + if _, err := fp.SendEvent(context.Background(), EventEmail{Kind: "k", Recipient: addr}); err != nil { + t.Fatalf("SendEvent = %v", err) + } + out := buf.String() + if strings.Contains(out, addr) { + t.Errorf("log output leaked raw recipient %q:\n%s", addr, out) + } + if !strings.Contains(out, maskEmail(addr)) { + t.Errorf("log output missing masked recipient %q:\n%s", maskEmail(addr), out) + } +} + +// TestFailover_AllFailMasksEmail — the all_failed (ERROR) path must also mask. +func TestFailover_AllFailMasksEmail(t *testing.T) { + var buf bytes.Buffer + orig := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) + defer slog.SetDefault(orig) + + fp, _ := NewFailoverProvider( + &fakeProvider{name: "brevo", err: transientErr()}, + &fakeProvider{name: "ses", err: transientErr()}, + ) + const addr = "bob@example.com" + _, _ = fp.SendEvent(context.Background(), EventEmail{Kind: "k", Recipient: addr}) + if strings.Contains(buf.String(), addr) { + t.Errorf("all_failed log leaked raw recipient %q:\n%s", addr, buf.String()) + } +} + +// TestFailover_PrimaryOkMasksEmail — the DEBUG happy-path line masks too. +func TestFailover_PrimaryOkMasksEmail(t *testing.T) { + var buf bytes.Buffer + orig := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) + defer slog.SetDefault(orig) + + fp, _ := NewFailoverProvider(&fakeProvider{name: "brevo", msgID: "ok"}, &fakeProvider{name: "ses"}) + const addr = "carol@example.com" + _, _ = fp.SendEvent(context.Background(), EventEmail{Kind: "k", Recipient: addr}) + if strings.Contains(buf.String(), addr) { + t.Errorf("primary_ok log leaked raw recipient %q:\n%s", addr, buf.String()) + } +} + +// TestFailover_PropagatesEventToProviders — the same EventEmail reaches each +// provider unchanged (no mutation as it walks the chain). +func TestFailover_PropagatesEventToProviders(t *testing.T) { + primary := &fakeProvider{name: "brevo", err: transientErr()} + fallback := &fakeProvider{name: "ses", msgID: "x"} + fp, _ := NewFailoverProvider(primary, fallback) + evt := EventEmail{Kind: "subscription.upgraded", Recipient: "u@example.com", Subject: "Hi"} + _, _ = fp.SendEvent(context.Background(), evt) + if fallback.gotEvt.Kind != evt.Kind || fallback.gotEvt.Subject != evt.Subject { + t.Errorf("fallback got %+v; want %+v", fallback.gotEvt, evt) + } +} + +// ── Factory wiring (EMAIL_PROVIDER_FALLBACK) ──────────────────────────────── + +// TestFactory_NoFallback_ReturnsBareProvider — THE INERT-DEFAULT CONTRACT. +// With no EMAIL_PROVIDER_FALLBACK, NewProvider returns the bare single +// provider (concrete *BrevoProvider), NOT a FailoverProvider wrapper. This is +// the proof of zero behavior change. +func TestFactory_NoFallback_ReturnsBareProvider(t *testing.T) { + p, err := NewProvider(Config{ + Provider: providerNameBrevo, + Brevo: BrevoConfig{APIKey: "k"}, + // Fallbacks deliberately nil. + }) + if err != nil { + t.Fatalf("NewProvider = %v", err) + } + if _, ok := p.(*FailoverProvider); ok { + t.Fatal("got *FailoverProvider with no fallback configured; want bare provider (inert default broken)") + } + if _, ok := p.(*BrevoProvider); !ok { + t.Errorf("got %T; want *BrevoProvider", p) + } +} + +// TestFactory_WithFallback_ReturnsFailover — a configured fallback yields a +// FailoverProvider with the right ordered names. +func TestFactory_WithFallback_ReturnsFailover(t *testing.T) { + p, err := NewProvider(Config{ + Provider: providerNameBrevo, + Brevo: BrevoConfig{APIKey: "k"}, + SES: SESConfig{AWSRegion: "us-east-1", AWSAccessKey: "a", AWSSecretKey: "s", FromEmail: "f@x.com"}, + Fallbacks: []string{providerNameSES}, + }) + if err != nil { + t.Fatalf("NewProvider = %v", err) + } + fp, ok := p.(*FailoverProvider) + if !ok { + t.Fatalf("got %T; want *FailoverProvider", p) + } + if got := fp.Name(); got != "failover(brevo,ses)" { + t.Errorf("Name() = %q; want failover(brevo,ses)", got) + } +} + +// TestFactory_UnknownFallback_Errors — a typo in EMAIL_PROVIDER_FALLBACK is a +// hard boot error (same strictness as the primary switch). +func TestFactory_UnknownFallback_Errors(t *testing.T) { + _, err := NewProvider(Config{ + Provider: providerNameBrevo, + Brevo: BrevoConfig{APIKey: "k"}, + Fallbacks: []string{"loops"}, + }) + if err == nil { + t.Fatal("NewProvider with unknown fallback = nil; want hard boot error") + } +} + +// TestFactory_FallbackMissingCreds_Skipped — a VALID fallback name whose creds +// are unset (SES_* not wired) is skipped-with-warning, NOT a boot crash. With +// SES the only fallback and it skipped, the factory returns the bare primary. +func TestFactory_FallbackMissingCreds_Skipped(t *testing.T) { + p, err := NewProvider(Config{ + Provider: providerNameBrevo, + Brevo: BrevoConfig{APIKey: "k"}, + SES: SESConfig{}, // no region/creds → NewSESProvider errors + Fallbacks: []string{providerNameSES}, + }) + if err != nil { + t.Fatalf("NewProvider = %v; want nil (missing-creds fallback should be skipped, not fatal)", err) + } + if _, ok := p.(*FailoverProvider); ok { + t.Error("got *FailoverProvider; want bare primary (only fallback was skipped)") + } + if _, ok := p.(*BrevoProvider); !ok { + t.Errorf("got %T; want *BrevoProvider (primary survives skipped fallback)", p) + } +} + +// TestFactory_FallbackWithBlankEntries_Tolerated — empty segments in the list +// (from "ses," or " , ses") are ignored, not treated as unknown providers. +func TestFactory_FallbackWithBlankEntries_Tolerated(t *testing.T) { + p, err := NewProvider(Config{ + Provider: providerNameBrevo, + Brevo: BrevoConfig{APIKey: "k"}, + SES: SESConfig{AWSRegion: "us-east-1", AWSAccessKey: "a", AWSSecretKey: "s", FromEmail: "f@x.com"}, + Fallbacks: []string{"", providerNameSES, " "}, + }) + if err != nil { + t.Fatalf("NewProvider = %v", err) + } + if _, ok := p.(*FailoverProvider); !ok { + t.Errorf("got %T; want *FailoverProvider", p) + } +} + +// TestFactory_PrimaryUnknown_StillErrors — the primary switch keeps its +// strictness through the buildOne refactor. +func TestFactory_PrimaryUnknown_StillErrors(t *testing.T) { + if _, err := NewProvider(Config{Provider: "loops"}); err == nil { + t.Fatal("NewProvider(loops primary) = nil; want error") + } +} diff --git a/internal/jobs/workers.go b/internal/jobs/workers.go index 5e82dde..7387140 100644 --- a/internal/jobs/workers.go +++ b/internal/jobs/workers.go @@ -360,6 +360,10 @@ func StartWorkers(ctx context.Context, db *sql.DB, rdb *redis.Client, cfg *confi FromEmail: cfg.SESFromEmail, TemplateNames: cfg.SESTemplateNames, }, + // Inert by default (EMAIL_PROVIDER_FALLBACK unset → nil → single + // provider). When set, NewProvider wraps the primary + named fallbacks + // in a FailoverProvider. See email.Config.Fallbacks. + Fallbacks: cfg.EmailProviderFallback, }) if err != nil { slog.Error("jobs.workers.email_provider_init_failed", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 6f874ce..8ed991d 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -459,6 +459,39 @@ var ( Help: "Event-email forwarder hits on an audit_log row whose kind has a builder but no Go renderer. A non-zero rate means a kind is being silently dropped — fix the eventEmailBodyRenderers map.", }, []string{"kind"}) + // ── Failover-ESP engagement counter (task #35) ─────────────────────────── + // + // Tracks the FailoverProvider's per-send outcome so an operator can see + // at a glance whether the primary ESP (Brevo) is carrying the traffic or + // the secondary (SES) is being pressed into service. This is the live + // signal that hardens the standing P0 (Brevo sender unvalidated → every + // send rejected): a sustained `fallback_ok` rate means the primary is + // degraded but email is still flowing via the secondary; any `all_failed` + // means BOTH providers refused the send and the email is being lost (the + // forwarder holds its cursor on the last Transient class, but a stream of + // all_failed is a data-loss-grade outage). + // + // Inert by default: the FailoverProvider is only constructed when + // EMAIL_PROVIDER_FALLBACK is set. With no secondary configured the worker + // uses the bare single provider and this counter never observes a label + // (lazy *Vec — absent from /metrics until first emit, per CLAUDE.md + // rule 25's eager-vs-lazy note). + // + // Label `outcome`: + // primary_ok — the first (primary) provider succeeded; no fallback engaged. + // fallback_ok — the primary failed/skipped and a later provider succeeded. + // all_failed — every provider in the chain failed; last error returned. + // + // NR alerts (shipped in the infra PR per rule 25): + // P1: rate(instant_email_failover_total{outcome="fallback_ok"}[10m]) > 0 + // sustained → primary ESP degraded, secondary carrying load. + // P0: rate(instant_email_failover_total{outcome="all_failed"}[5m]) > 0 + // → both ESPs refusing; email being lost. + EmailFailoverTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "instant_email_failover_total", + Help: "FailoverProvider per-send outcome. Labelled by outcome (primary_ok|fallback_ok|all_failed). A non-zero fallback_ok rate means the primary ESP is degraded; all_failed means both ESPs refused and email is being lost.", + }, []string{"outcome"}) + // ── propagation_runner — unexpected_skip counter (CHAOS-DRILL-2026-05-20 F1) ─ // // Every time the propagation_runner's per-resource RegradeResource call