Add multi-source reconciliation to billing event ledger

Author: tjementum

application/account/BackOffice/routes/billing-events/-components/BillingEventsToolbar.tsx

@@ -17,7 +17,10 @@ interface BillingEventsToolbarProps {
 }
 
 // Order matches the BillingEventType enum so the dropdown reads in the same lifecycle order operators
-// see in our domain log (creation → renewal → upgrade → downgrade → cancellation → payment).
+// see in our domain log (creation → renewal → upgrade → downgrade → cancellation → payment → audit).
+// IMPORTANT: this list mirrors the C# BillingEventType enum (see application/account/Core/Features/
+// Subscriptions/Domain/BillingEvent.cs). Add new enum values here too — the enum's doc comment also
+// flags this requirement.
 const ALL_EVENT_TYPES: BillingEventType[] = [
   BillingEventType.SubscriptionCreated,
   BillingEventType.SubscriptionRenewed,
@@ -30,12 +33,15 @@ const ALL_EVENT_TYPES: BillingEventType[] = [
   BillingEventType.SubscriptionExpired,
   BillingEventType.SubscriptionImmediatelyCancelled,
   BillingEventType.SubscriptionSuspended,
+  BillingEventType.SubscriptionPastDue,
   BillingEventType.PaymentFailed,
   BillingEventType.PaymentRecovered,
   BillingEventType.PaymentRefunded,
   BillingEventType.BillingInfoAdded,
   BillingEventType.BillingInfoUpdated,
-  BillingEventType.PaymentMethodUpdated
+  BillingEventType.PaymentMethodUpdated,
+  BillingEventType.NoOp,
+  BillingEventType.Unclassified
 ];
 
 export function BillingEventsToolbar({ search, eventTypes }: Readonly<BillingEventsToolbarProps>) {

application/account/BackOffice/shared/lib/api/labels.ts

@@ -124,6 +124,8 @@ export function getBillingEventTypeLabel(type: BillingEventType): string {
       return t`Cancelled immediately`;
     case BillingEventType.SubscriptionSuspended:
       return t`Suspended`;
+    case BillingEventType.SubscriptionPastDue:
+      return t`Past due`;
     case BillingEventType.PaymentFailed:
       return t`Payment failed`;
     case BillingEventType.PaymentRecovered:

application/account/BackOffice/shared/lib/billingEventStyle.ts

@@ -72,6 +72,10 @@ export const BILLING_EVENT_VARIANT: Record<BillingEventType, BillingEventVariant
     className: "bg-rose-500/10 text-rose-500 border-rose-500/20",
     icon: PauseCircleIcon
   },
+  [BillingEventType.SubscriptionPastDue]: {
+    className: "bg-amber-500/10 text-amber-600 border-amber-500/30",
+    icon: CircleAlertIcon
+  },
   [BillingEventType.PaymentFailed]: {
     className: "bg-rose-500/10 text-rose-500 border-rose-500/20",
     icon: CircleAlertIcon

application/account/BackOffice/shared/translations/locale/da-DK.po

@@ -570,6 +570,9 @@ msgstr "Siden blev ikke fundet"
 msgid "Paid"
 msgstr "Betalt"
 
+msgid "Past due"
+msgstr "Forfalden"
+
 msgid "Payment failed"
 msgstr "Betaling mislykkedes"
 

application/account/BackOffice/shared/translations/locale/en-US.po

@@ -570,6 +570,9 @@ msgstr "Page not found"
 msgid "Paid"
 msgstr "Paid"
 
+msgid "Past due"
+msgstr "Past due"
+
 msgid "Payment failed"
 msgstr "Payment failed"
 

…000_AddBillingEventsAndDriftDetection.cs …000_AddBillingEventsAndDriftDetection.csapplication/account/Core/Database/Migrations/20260509120000_AddBillingEventsAndDriftDetection.cs renamed to application/account/Core/Database/Migrations/20260509180000_AddBillingEventsAndDriftDetection.cs application/account/Core/Database/Migrations/20260509120000_AddBillingEventsAndDriftDetection.cs renamed to application/account/Core/Database/Migrations/20260509180000_AddBillingEventsAndDriftDetection.cs

@@ -4,21 +4,18 @@
 namespace Account.Database.Migrations;
 
 [DbContext(typeof(AccountDbContext))]
-[Migration("20260509120000_AddBillingEventsAndDriftDetection")]
+[Migration("20260509180000_AddBillingEventsAndDriftDetection")]
 public sealed class AddBillingEventsAndDriftDetection : Migration
 {
     protected override void Up(MigrationBuilder migrationBuilder)
     {
-        // Subscription drift columns. IF NOT EXISTS so this migration is idempotent on staging where an
-        // earlier iteration of this migration already added them. Removed before merging to main; new
-        // environments only see plain ADD COLUMN.
-        migrationBuilder.Sql("ALTER TABLE subscriptions ADD COLUMN IF NOT EXISTS subscribed_since timestamptz;");
-        migrationBuilder.Sql("ALTER TABLE subscriptions ADD COLUMN IF NOT EXISTS scheduled_price_amount numeric(18,2);");
-        migrationBuilder.Sql("ALTER TABLE subscriptions ADD COLUMN IF NOT EXISTS has_drift_detected boolean NOT NULL DEFAULT false;");
-        migrationBuilder.Sql("ALTER TABLE subscriptions ADD COLUMN IF NOT EXISTS drift_checked_at timestamptz;");
-        migrationBuilder.Sql("ALTER TABLE subscriptions ADD COLUMN IF NOT EXISTS drift_discrepancies jsonb NOT NULL DEFAULT '[]';");
+        migrationBuilder.AddColumn<DateTimeOffset>("subscribed_since", "subscriptions", "timestamptz", nullable: true);
+        migrationBuilder.AddColumn<decimal>("scheduled_price_amount", "subscriptions", "numeric(18,2)", nullable: true);
+        migrationBuilder.AddColumn<bool>("has_drift_detected", "subscriptions", "boolean", nullable: false, defaultValue: false);
+        migrationBuilder.AddColumn<DateTimeOffset>("drift_checked_at", "subscriptions", "timestamptz", nullable: true);
+        migrationBuilder.AddColumn<string>("drift_discrepancies", "subscriptions", "jsonb", nullable: false, defaultValue: "[]");
 
-        migrationBuilder.Sql("CREATE INDEX IF NOT EXISTS ix_subscriptions_has_drift_detected ON subscriptions (has_drift_detected) WHERE has_drift_detected = true;");
+        migrationBuilder.CreateIndex("ix_subscriptions_has_drift_detected", "subscriptions", "has_drift_detected", filter: "has_drift_detected = true");
 
         // Subscriptions created before this migration have no subscribed_since because the column did not
         // exist when the Basis -> paid transition occurred. Best available proxy for the start of their paid
@@ -55,25 +52,22 @@ WHERE jsonb_array_length(payment_transactions) > 0
             """
         );
 
-        // Add the check constraint only if it doesn't exist. Removed before merging to main; new
-        // environments only see a plain ALTER TABLE … ADD CONSTRAINT.
-        migrationBuilder.Sql(
-            """
-            DO $$
-            BEGIN
-                IF NOT EXISTS (
-                    SELECT 1 FROM pg_constraint
-                    WHERE conname = 'chk_subscriptions_payment_transactions_tax_breakdown'
-                      AND conrelid = 'subscriptions'::regclass
-                ) THEN
-                    ALTER TABLE subscriptions
-                    ADD CONSTRAINT chk_subscriptions_payment_transactions_tax_breakdown
-                    CHECK (NOT jsonb_path_exists(payment_transactions, '$[*] ? (!(@.AmountExcludingTax.type() == "number") || !(@.TaxAmount.type() == "number"))'));
-                END IF;
-            END $$;
-            """
+        migrationBuilder.AddCheckConstraint(
+            "chk_subscriptions_payment_transactions_tax_breakdown",
+            "subscriptions",
+            """NOT jsonb_path_exists(payment_transactions, '$[*] ? (!(@.AmountExcludingTax.type() == "number") || !(@.TaxAmount.type() == "number"))')"""
         );
 
+        // The billing_events table is append-only. The unique index on stripe_event_id enforces strict
+        // 1:1 with Stripe events: every recognized Stripe event yields exactly one row. Stripe's events.list
+        // API has a 30-day retention window (see https://docs.stripe.com/api/events), so the local
+        // stripe_events table is the authoritative source for replays beyond that window.
+        // Hard rule: NO migration ever drops, deletes from, or truncates this table. Schema changes use
+        // ALTER TABLE ADD/DROP COLUMN. Forensics and audit depend on full history being preserved.
+        // tenant_id is the soft-scope query filter for ITenantScopedEntity; no FK to tenants because the
+        // back-office is cross-tenant by design and uses IgnoreQueryFilters([QueryFilterNames.Tenant]).
+        // modified_at is inherited from the framework's AggregateRoot shape and remains NULL by design —
+        // billing_events is append-only forever (rows are never updated after insert).
         migrationBuilder.CreateTable(
             "billing_events",
             table => new
@@ -103,5 +97,20 @@ ADD CONSTRAINT chk_subscriptions_payment_transactions_tax_breakdown
         migrationBuilder.CreateIndex("ix_billing_events_tenant_id_occurred_at", "billing_events", ["tenant_id", "occurred_at"], descending: [false, true]);
         migrationBuilder.CreateIndex("ix_billing_events_occurred_at", "billing_events", "occurred_at", descending: [true]);
         migrationBuilder.CreateIndex("ix_billing_events_subscription_id", "billing_events", "subscription_id");
+
+        // stripe_events extensions for the multi-source reconciliation architecture:
+        // - api_version: pinned at event creation per https://docs.stripe.com/api/events; lets the
+        //   replayer dispatch to the correct payload resolver when Stripe ships a new API version.
+        // - payload_hash: SHA-256 of the raw payload at first observation; lets AcknowledgeStripeWebhook
+        //   detect StripeEventPayloadDivergence (same id, different payload) without comparing JSON bodies.
+        // - recovered_at / recovery_source: non-null when the event was added by reconciliation
+        //   (events.list or webhook_endpoint_deliveries) rather than via webhook delivery — forensic
+        //   marker that a webhook delivery was missed.
+        migrationBuilder.AddColumn<string>("api_version", "stripe_events", "text", nullable: true);
+        migrationBuilder.AddColumn<DateTimeOffset>("recovered_at", "stripe_events", "timestamptz", nullable: true);
+        migrationBuilder.AddColumn<string>("recovery_source", "stripe_events", "text", nullable: true);
+        migrationBuilder.AddColumn<string>("payload_hash", "stripe_events", "text", nullable: true);
+
+        migrationBuilder.CreateIndex("ix_stripe_events_recovered_at", "stripe_events", "recovered_at", filter: "recovered_at IS NOT NULL");
     }
 }

application/account/Core/Features/Subscriptions/Commands/AcknowledgeStripeWebhook.cs

@@ -1,7 +1,9 @@
 using Account.Features.Subscriptions.Domain;
+using Account.Features.Subscriptions.Shared;
 using Account.Integrations.Stripe;
 using JetBrains.Annotations;
 using SharedKernel.Cqrs;
+using SharedKernel.Telemetry;
 
 namespace Account.Features.Subscriptions.Commands;
 
@@ -15,7 +17,9 @@ public sealed record AcknowledgeStripeWebhookCommand(string Payload, string Sign
 public sealed class AcknowledgeStripeWebhookHandler(
     IStripeEventRepository stripeEventRepository,
     StripeClientFactory stripeClientFactory,
-    TimeProvider timeProvider
+    ITelemetryEventsCollector events,
+    TimeProvider timeProvider,
+    ILogger<AcknowledgeStripeWebhookHandler> logger
 ) : IRequestHandler<AcknowledgeStripeWebhookCommand, Result<StripeCustomerId?>>
 {
     public async Task<Result<StripeCustomerId?>> Handle(AcknowledgeStripeWebhookCommand command, CancellationToken cancellationToken)
@@ -27,15 +31,31 @@ TimeProvider timeProvider
             return Result<StripeCustomerId?>.BadRequest("Invalid webhook signature.");
         }
 
-        if (await stripeEventRepository.ExistsAsync(webhookEvent.EventId, cancellationToken))
+        var payloadHash = StripeEventPayloadHasher.Hash(command.Payload);
+
+        // Idempotency: Stripe redelivers webhooks on transient errors (network, our 5xx, etc.). Same event
+        // id arriving twice with the same payload is a no-op. Same id with a *different* payload is a
+        // forensic anomaly: the existing row is preserved unchanged and a divergence telemetry event is
+        // emitted so the drift banner can surface it. We never overwrite stripe_events rows.
+        var existing = await stripeEventRepository.GetByIdAsync(StripeEventId.NewId(webhookEvent.EventId), cancellationToken);
+        if (existing is not null)
         {
+            if (existing.PayloadHash is not null && existing.PayloadHash != payloadHash)
+            {
+                logger.LogWarning(
+                    "Stripe event {EventId} arrived twice with different payloads (existing hash {ExistingHash} vs new {NewHash}); existing row preserved",
+                    webhookEvent.EventId, existing.PayloadHash, payloadHash
+                );
+                events.CollectEvent(new StripeEventPayloadMismatch(webhookEvent.EventId, webhookEvent.EventType, existing.PayloadHash, payloadHash));
+            }
+
             return Result<StripeCustomerId?>.Success(webhookEvent.CustomerId);
         }
 
         var now = timeProvider.GetUtcNow();
         var customerId = webhookEvent.CustomerId;
 
-        var stripeEvent = StripeEvent.Create(webhookEvent.EventId, webhookEvent.EventType, customerId, command.Payload);
+        var stripeEvent = StripeEvent.Create(webhookEvent.EventId, webhookEvent.EventType, customerId, command.Payload, webhookEvent.ApiVersion, payloadHash);
 
         if (customerId is null)
         {

application/account/Core/Features/Subscriptions/Domain/BillingEvent.cs

@@ -26,6 +26,12 @@ public override string ToString()
 ///     subscription's drift flag for admin review.
 ///     Idempotent on <see cref="StripeEventId" /> (unique index): redelivered webhooks and re-pulls from
 ///     the Stripe events API are no-ops.
+///     Source of truth: the local <c>stripe_events</c> archive, NOT Stripe's events.list API. Stripe only
+///     retains events for 30 days (see https://docs.stripe.com/api/events) — anything older must come from
+///     our local archive. The events.list API is used only as a reconciliation source for detecting
+///     webhooks that never reached us within the retention window.
+///     Hard rule: rows in this table are never deleted, never updated. Schema changes use ALTER TABLE
+///     ADD/DROP COLUMN, never DROP/TRUNCATE/DELETE FROM.
 /// </summary>
 public sealed class BillingEvent : AggregateRoot<BillingEventId>, ITenantScopedEntity
 {
@@ -101,6 +107,13 @@ public static BillingEvent Create(
     }
 }
 
+/// <summary>
+///     The type of subscription-relevant Stripe event recorded by the BillingEvent log.
+///     IMPORTANT: when adding a new value, also add it to the multi-select on /billing-events
+///     (see <c>application/account/BackOffice/routes/billing-events/-components/BillingEventsToolbar.tsx</c>,
+///     constant <c>ALL_EVENT_TYPES</c>). The toolbar is hand-maintained and does not enumerate the
+///     enum at runtime — operators won't be able to filter by a new type until that list is updated.
+/// </summary>
 [PublicAPI]
 [JsonConverter(typeof(JsonStringEnumConverter))]
 public enum BillingEventType
@@ -116,6 +129,15 @@ public enum BillingEventType
     SubscriptionExpired,
     SubscriptionImmediatelyCancelled,
     SubscriptionSuspended,
+
+    /// <summary>
+    ///     Stripe transitioned the subscription's status from active to past_due (a payment failed).
+    ///     Fires alongside <see cref="PaymentFailed" /> from the corresponding invoice.payment_failed event;
+    ///     pairs with <see cref="SubscriptionReactivated" /> when payment recovers and status returns to active.
+    ///     Carries forward CommittedMrr unchanged and AmountDelta=null — the customer is still on the plan,
+    ///     just behind on payment.
+    /// </summary>
+    SubscriptionPastDue,
     PaymentFailed,
     PaymentRecovered,
     PaymentRefunded,