fix(bridge): quarantine undeliverable refunds instead of crashlooping (#1092)

A refund whose Stellar target is permanently undeliverable (e.g. the
target removed its trustline) caused submitTransaction to return a fatal
error, exiting the bridge. The on-chain on_finalize retry then re-emitted
the refund every RetryInterval blocks, producing an endless crash loop
(observed: 10k+ restarts over ~40 days).

Make refund submission non-fatal:
- Classify Stellar payment result codes. Target-side, bridge-unresolvable
  failures (op_no_trust, op_no_destination, op_not_authorized) are
  forfeited: the refund is marked executed on chain to stop the retry
  loop, with a refund_quarantined alert.
- All other submit failures (incl. op_line_full and transient codes) are
  postponed and retried via the on-chain re-emit, matching the existing
  withdraw path.
- op_underfunded (bridge wallet empty) raises a bridge_account_underfunded
  vault alert at submit time, covering both refund and withdraw paths.

Adds unit tests for the result-code classifiers and documents the two new
structured-logging events.

Refs #1089

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: sameh-farouk

bridge/docs/observability.md

@@ -60,6 +60,7 @@ For example, if a customer is complaining that their deposit never bridged, you
 - `refund_skipped`: a refund request skipped by the bridge instance as it has already been refunded.
 - `refund_proposed`: a refund has proposed or signed by the bridge instance.
 - `refund_postponed`: a refund has postponed due to a problem in sending this transaction to the stellar network and will be retried later.
+- `refund_quarantined`: a refund is permanently undeliverable to the target account (e.g. the target removed its trustline, the account no longer exists, or it is not authorized to hold the asset) and has been marked executed on chain so the bridge stops retrying it. The refunded TFT is forfeited.
 - `refund_completed`: a refund has completed and received on the target stellar account.
 
 ##### Withdraw related 
@@ -74,6 +75,7 @@ For example, if a customer is complaining that their deposit never bridged, you
 ##### Bridge vault account related
 - `payment_received` : This event represents successful payment to the bridge account (a deposit).
 - `stellar_transaction_submitted` : This event represents successful transaction from the bridge account (a refund or a withdraw).
+- `bridge_account_underfunded` : The bridge account has insufficient funds to submit a transaction. The transaction is retried, but the bridge wallet must be refilled by an operator.
 
 ### Metrics:
 
@@ -659,6 +661,38 @@ the source field set contains all fields which are included in the source object
     </tbody>
 </table>
 
+##### refund_quarantined
+
+- kind: alert
+
+- category: refund
+
+<table>
+    <thead>
+        <tr><th colspan="4"><div>refund_quarantined Event Properties</div></th></tr>
+        <tr>
+            <th>Property</th>
+            <th>Type</th>
+            <th>Required</th>
+            <th>Description</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td><code>target</code></td>
+            <td>string</td>
+            <td>yes</td>
+            <td>The stellar target account the refund could not be delivered to. </td>
+        </tr>
+        <tr>
+            <td><code>amount</code></td>
+            <td>number</td>
+            <td>yes</td>
+            <td>The forfeited refund amount in stroops. </td>
+        </tr>
+    </tbody>
+</table>
+
 ##### event_burn_tx_created_received
 
 - kind: event
@@ -908,6 +942,29 @@ the source field set contains all fields which are included in the source object
     </tbody>
 </table>
 
+##### bridge_account_underfunded
+
+- kind: alert
+
+- category: vault
+
+<table>
+    <thead>
+        <tr><th colspan="4"><div>bridge_account_underfunded Event Properties</div></th></tr>
+        <tr>
+            <th>Property</th>
+            <th>Type</th>
+            <th>Required</th>
+            <th>Description</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td colspan="4"><div>No metadata</div></td>
+        </tr>            
+    </tbody>
+</table>
+
 ##### wallet_balance
 
 - kind: metric

bridge/tfchain_bridge/pkg/bridge/refund.go

@@ -100,7 +100,43 @@ func (bridge *Bridge) handleRefundReady(ctx context.Context, refundReadyEvent su
 
 	// Todo, retry here?
 	if err = bridge.wallet.CreateRefundPaymentWithSignaturesAndSubmit(ctx, refund.Target, uint64(refund.Amount), refund.TxHash, refund.Signatures, int64(refund.SequenceNumber)); err != nil {
-		return err
+		// A refund submission must never crash the bridge. A single failure would
+		// otherwise propagate up to a fatal exit, and the on-chain on_finalize
+		// retry would re-emit the event every RetryInterval blocks, producing an
+		// endless crash loop.
+		if errors.Is(err, pkg.ErrStellarTransactionUndeliverable) {
+			// Target-side, bridge-unresolvable (e.g. the target removed its Stellar
+			// trustline). It can never be delivered, so quarantine it by marking it
+			// executed on chain to stop the on-chain retries. The refunded TFT is
+			// forfeited.
+			logger.Warn().
+				Err(err).
+				Str("event_action", "refund_quarantined").
+				Str("event_kind", "alert").
+				Str("category", "refund").
+				Dict("metadata", zerolog.Dict().
+					Str("target", refund.Target).
+					Uint64("amount", uint64(refund.Amount))).
+				Msg("the refund is permanently undeliverable to the target account and has been quarantined; the bridge will continue")
+
+			if execErr := bridge.subClient.RetrySetRefundTransactionExecutedTx(ctx, refund.TxHash); execErr != nil {
+				return execErr
+			}
+			return nil
+		}
+
+		// Any other failure (transient network/sequence issue, or a recoverable
+		// target condition such as op_line_full) is postponed: log and continue,
+		// relying on the on-chain on_finalize retry to re-emit the event so we try
+		// again later. Matches handleWithdrawReady's behaviour.
+		logger.Info().
+			Str("event_action", "refund_postponed").
+			Str("event_kind", "event").
+			Str("category", "refund").
+			Dict("metadata", zerolog.Dict().
+				Str("reason", err.Error())).
+			Msg("the refund has been postponed due to a problem submitting the transaction to the Stellar network; it will be retried")
+		return nil
 	}
 
 	err = bridge.subClient.RetrySetRefundTransactionExecutedTx(ctx, refund.TxHash)

bridge/tfchain_bridge/pkg/config.go

@@ -30,3 +30,9 @@ var ErrTransactionAlreadyRefunded = errors.New("transaction is already refunded"
 var ErrTransactionAlreadyMinted = errors.New("transaction is already minted")
 var ErrTransactionAlreadyBurned = errors.New("transaction is already burned")
 var ErrNoSignatures = errors.New("transaction has no signatures")
+
+// ErrStellarTransactionUndeliverable signals that a Stellar submission failed
+// with a permanent, target-side error (e.g. the destination removed its
+// trustline) that can never succeed on retry. The refund must be quarantined
+// and the bridge must continue, rather than crashing in a retry loop.
+var ErrStellarTransactionUndeliverable = errors.New("stellar transaction permanently undeliverable to target account")

bridge/tfchain_bridge/pkg/stellar/stellar.go

@@ -276,6 +276,58 @@ func (w *StellarWallet) createTransaction(ctx context.Context, txn txnbuild.Tran
 	return tx, nil
 }
 
+// terminalSubmitOperationCodes are Horizon payment operation result codes that
+// represent a target-side failure the bridge can never resolve on its own:
+// delivery requires a deliberate action by the destination owner (add a
+// trustline, create the account, get issuer authorization) that the bridge has
+// no visibility into. A refund hitting one of these is forfeited rather than
+// retried forever.
+//
+// Deliberately excluded:
+//   - op_line_full: the destination CAN receive the asset but is momentarily at
+//     its trustline limit; it may succeed once the balance is spent down, so it
+//     is postponed and retried rather than forfeited.
+//   - op_underfunded / op_src_no_trust / op_src_not_authorized: bridge-side
+//     (source) problems that must alert/halt, never forfeit a user's refund.
+//   - op_malformed / op_no_issuer: bridge bug or global asset misconfiguration
+//     affecting all transactions, not a per-target condition.
+var terminalSubmitOperationCodes = map[string]struct{}{
+	"op_no_trust":       {}, // destination has no trustline for the asset
+	"op_no_destination": {}, // destination account does not exist
+	"op_not_authorized": {}, // destination is not authorized to hold the asset
+}
+
+// isTerminalSubmitError reports whether a Stellar submission failure is a
+// target-side error that the bridge cannot resolve by retrying, and so should
+// be forfeited rather than postponed.
+func isTerminalSubmitError(codes *hProtocol.TransactionResultCodes) bool {
+	if codes == nil {
+		return false
+	}
+	for _, op := range codes.OperationCodes {
+		if _, ok := terminalSubmitOperationCodes[op]; ok {
+			return true
+		}
+	}
+	return false
+}
+
+// isInsufficientFundsError reports whether a Stellar submission failed because
+// the bridge (source) account is out of funds. This is an operational
+// condition that requires the bridge wallet to be refilled; it is not the
+// target's fault and the transaction must not be forfeited.
+func isInsufficientFundsError(codes *hProtocol.TransactionResultCodes) bool {
+	if codes == nil {
+		return false
+	}
+	for _, op := range codes.OperationCodes {
+		if op == "op_underfunded" {
+			return true
+		}
+	}
+	return false
+}
+
 func (w *StellarWallet) submitTransaction(ctx context.Context, txn *txnbuild.Transaction) error {
 	client, err := w.getHorizonClient()
 	if err != nil {
@@ -287,8 +339,26 @@ func (w *StellarWallet) submitTransaction(ctx context.Context, txn *txnbuild.Tra
 	if err != nil {
 		log.Info().Msg(err.Error())
 		if hError, ok := err.(*horizonclient.Error); ok {
-			if ok {
-				log.Err(err).Msgf("error while submitting transaction %+v", hError.Problem.Extras)
+			log.Err(err).Msgf("error while submitting transaction %+v", hError.Problem.Extras)
+			if codes, rcErr := hError.ResultCodes(); rcErr == nil {
+				// A target-side failure (e.g. the destination removed its trustline)
+				// can never succeed on retry. Surface it as a typed error so the
+				// caller can quarantine the transaction instead of retrying forever.
+				// The account sequence is irrelevant here, so don't reset it.
+				if isTerminalSubmitError(codes) {
+					return errors.Wrapf(pkg.ErrStellarTransactionUndeliverable, "operation result codes %v", codes.OperationCodes)
+				}
+				// The bridge account is out of funds. This is recoverable (the
+				// transaction will be retried) but needs operator attention to refill
+				// the bridge wallet, so raise an alert rather than only postponing.
+				if isInsufficientFundsError(codes) {
+					log.Warn().
+						Str("trace_id", fmt.Sprint(ctx.Value(TraceIdKey{}))).
+						Str("event_action", "bridge_account_underfunded").
+						Str("event_kind", "alert").
+						Str("category", "vault").
+						Msg("the bridge account has insufficient funds to submit the transaction; it will be retried but the bridge wallet must be refilled")
+				}
 			}
 		}
 		errSequence := w.resetAccountSequence()

bridge/tfchain_bridge/pkg/stellar/stellar_test.go

@@ -0,0 +1,139 @@
+package stellar
+
+import (
+	"testing"
+
+	hProtocol "github.com/stellar/go/protocols/horizon"
+)
+
+func TestIsTerminalSubmitError(t *testing.T) {
+	tests := []struct {
+		name  string
+		codes *hProtocol.TransactionResultCodes
+		want  bool
+	}{
+		{
+			name:  "nil codes are not terminal",
+			codes: nil,
+			want:  false,
+		},
+		{
+			name: "op_no_trust is terminal (target removed trustline)",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_no_trust"},
+			},
+			want: true,
+		},
+		{
+			name: "op_no_destination is terminal (target account gone)",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_no_destination"},
+			},
+			want: true,
+		},
+		{
+			name: "op_not_authorized is terminal (target not authorized to hold the asset)",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_not_authorized"},
+			},
+			want: true,
+		},
+		{
+			name: "op_line_full is NOT terminal (target trustline momentarily full; may succeed on retry - postpone, do not forfeit)",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_line_full"},
+			},
+			want: false,
+		},
+		{
+			name: "op_underfunded is NOT terminal (bridge wallet out of funds - operational, must alert not forfeit)",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_underfunded"},
+			},
+			want: false,
+		},
+		{
+			name: "op_src_no_trust is NOT terminal (bridge-side trustline missing, not the target's fault)",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_src_no_trust"},
+			},
+			want: false,
+		},
+		{
+			name: "tx_bad_seq is NOT terminal (transient, resolved by sequence reset)",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_bad_seq",
+				OperationCodes:  nil,
+			},
+			want: false,
+		},
+		{
+			name: "terminal code mixed with others is still terminal",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_success", "op_no_trust"},
+			},
+			want: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := isTerminalSubmitError(tt.codes); got != tt.want {
+				t.Errorf("isTerminalSubmitError() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsInsufficientFundsError(t *testing.T) {
+	tests := []struct {
+		name  string
+		codes *hProtocol.TransactionResultCodes
+		want  bool
+	}{
+		{
+			name:  "nil codes are not an underfunded condition",
+			codes: nil,
+			want:  false,
+		},
+		{
+			name: "op_underfunded means the bridge wallet is out of funds",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_underfunded"},
+			},
+			want: true,
+		},
+		{
+			name: "op_no_trust is a target-side failure, not underfunded",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_no_trust"},
+			},
+			want: false,
+		},
+		{
+			name: "op_underfunded mixed with others is still underfunded",
+			codes: &hProtocol.TransactionResultCodes{
+				TransactionCode: "tx_failed",
+				OperationCodes:  []string{"op_success", "op_underfunded"},
+			},
+			want: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := isInsufficientFundsError(tt.codes); got != tt.want {
+				t.Errorf("isInsufficientFundsError() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}