Improve timeout handling and error recovery

- Enhanced Lightning Network error handling with specific timeout detection
- Made pathfinding and bot handler timeouts configurable via environment variables
- Implemented exponential backoff retry strategy for failed payments
- Added timeout monitoring and structured logging for better debugging
- Enhanced PendingPayment model with error tracking and retry scheduling

Fixes 60-second timeout issues that were causing bot unresponsiveness.

Author: grunch

.env-sample

@@ -61,6 +61,12 @@ LOG_LEVEL='debug'
 # Max routing fee that we want to pay to the network, 0.001 = 0.1%
 MAX_ROUTING_FEE=0.001
 
+# Lightning Network pathfinding timeout in milliseconds (default: 60000 = 60 seconds)
+LN_PATHFINDING_TIMEOUT=60000
+
+# Telegram bot handler timeout in milliseconds (default: 60000 = 60 seconds)
+BOT_HANDLER_TIMEOUT=60000
+
 # Attempts to pay the invoice again when the payment failed
 PAYMENT_ATTEMPTS=2
 

app.ts

@@ -25,7 +25,9 @@ import { CommunityContext } from "./bot/modules/community/communityContext";
   mongoose.connection
     .once('open', async () => {
       logger.info('Connected to Mongo instance.');
-      let options: Partial<Telegraf.Options<CommunityContext>> = { handlerTimeout: 60000 };
+      // Use configurable bot handler timeout, default to 60 seconds
+      const handlerTimeout = parseInt(process.env.BOT_HANDLER_TIMEOUT || '60000');
+      let options: Partial<Telegraf.Options<CommunityContext>> = { handlerTimeout };
       if (process.env.SOCKS_PROXY_HOST) {
         const agent = new SocksProxyAgent(process.env.SOCKS_PROXY_HOST);
         options = {

jobs/pending_payments.ts

@@ -14,12 +14,21 @@ export const attemptPendingPayments = async (bot: Telegraf<CommunityContext>): P
     attempts: { $lt: process.env.PAYMENT_ATTEMPTS },
     is_invoice_expired: false,
     community_id: null,
+    next_retry: { $lte: new Date() },
   });
   for (const pending of pendingPayments) {
     const order = await Order.findOne({ _id: pending.order_id });
     try {
       if (order === null) throw Error("Order was not found in DB");
       pending.attempts++;
+      
+      // Calculate exponential backoff delay
+      const baseDelay = 5 * 60 * 1000; // 5 minutes
+      const exponentialDelay = baseDelay * Math.pow(2, pending.attempts - 1);
+      const maxDelay = 60 * 60 * 1000; // 1 hour max
+      const nextRetryDelay = Math.min(exponentialDelay, maxDelay);
+      pending.next_retry = new Date(Date.now() + nextRetryDelay);
+      
       if (order.status === 'SUCCESS') {
         pending.paid = true;
         await pending.save();
@@ -85,9 +94,25 @@ export const attemptPendingPayments = async (bot: Telegraf<CommunityContext>): P
         );
         await messages.rateUserMessage(bot, buyerUser, order, i18nCtx);
       } else {
+        // Enhanced error handling for different payment failure types
+        if (payment && typeof payment === 'object' && 'error' in payment) {
+          pending.last_error = payment.error as string;
+          
+          if (payment.error === 'TIMEOUT') {
+            logger.warn(`Payment timeout for order ${order._id}, attempt ${pending.attempts}`);
+          } else if (payment.error === 'ROUTING_FAILED') {
+            logger.warn(`Routing failed for order ${order._id}, attempt ${pending.attempts}`);
+          } else {
+            logger.error(`Payment failed for order ${order._id}, attempt ${pending.attempts}, error: ${payment.error}`);
+          }
+        } else {
+          pending.last_error = 'PAYMENT_FAILED';
+          logger.error(`Payment failed for order ${order._id}, attempt ${pending.attempts}`);
+        }
+        
         if (
           process.env.PAYMENT_ATTEMPTS !== undefined &&
-          pending.attempts === parseInt(process.env.PAYMENT_ATTEMPTS)
+          pending.attempts >= parseInt(process.env.PAYMENT_ATTEMPTS)
         ) {
           order.paid_hold_buyer_invoice_updated = false;
           await messages.toBuyerPendingPaymentFailedMessage(
@@ -124,11 +149,19 @@ export const attemptCommunitiesPendingPayments = async (bot: Telegraf<CommunityC
     attempts: { $lt: process.env.PAYMENT_ATTEMPTS },
     is_invoice_expired: false,
     community_id: { $ne: null },
+    next_retry: { $lte: new Date() },
   });
 
   for (const pending of pendingPayments) {
     try {
       pending.attempts++;
+      
+      // Calculate exponential backoff delay for community payments
+      const baseDelay = 5 * 60 * 1000; // 5 minutes
+      const exponentialDelay = baseDelay * Math.pow(2, pending.attempts - 1);
+      const maxDelay = 60 * 60 * 1000; // 1 hour max
+      const nextRetryDelay = Math.min(exponentialDelay, maxDelay);
+      pending.next_retry = new Date(Date.now() + nextRetryDelay);
 
       // We check if this new payment is on flight
       const isPending: boolean = await isPendingPayment(pending.payment_request);
@@ -174,9 +207,22 @@ export const attemptCommunitiesPendingPayments = async (bot: Telegraf<CommunityC
           })
         );
       } else {
+        // Enhanced error handling for community payments
+        if (payment && typeof payment === 'object' && 'error' in payment) {
+          pending.last_error = payment.error as string;
+          logger.error(
+            `Community ${community.id}: Withdraw failed after ${pending.attempts} attempts, amount ${pending.amount} sats, error: ${payment.error}`
+          );
+        } else {
+          pending.last_error = 'PAYMENT_FAILED';
+          logger.error(
+            `Community ${community.id}: Withdraw failed after ${pending.attempts} attempts, amount ${pending.amount} sats`
+          );
+        }
+        
         if (
           process.env.PAYMENT_ATTEMPTS !== undefined &&
-          pending.attempts === parseInt(process.env.PAYMENT_ATTEMPTS)
+          pending.attempts >= parseInt(process.env.PAYMENT_ATTEMPTS)
         ) {
           await bot.telegram.sendMessage(
             user.tg_id,
@@ -185,9 +231,6 @@ export const attemptCommunitiesPendingPayments = async (bot: Telegraf<CommunityC
             })
           );
         }
-        logger.error(
-          `Community ${community.id}: Withdraw failed after ${pending.attempts} attempts, amount ${pending.amount} sats`
-        );
       }
     } catch (error) {
       logger.error(`attemptCommunitiesPendingPayments catch error: ${error}`);

ln/pay_request.ts

@@ -8,7 +8,7 @@ import { User, PendingPayment } from '../models';
 import lnd from './connect';
 import { handleReputationItems, getUserI18nContext } from '../util';
 import * as messages from '../bot/messages';
-import { logger } from '../logger';
+import { logger, logTimeout, logOperationDuration } from '../logger';
 import * as OrderEvents from '../bot/modules/events/orders';
 import { IOrder } from '../models/order';
 import { HasTelegram } from '../bot/start';
@@ -24,6 +24,9 @@ interface PayViaPaymentRequestParams {
 }
 
 const payRequest = async ({ request, amount }: { request: string, amount: number }) => {
+  const startTime = Date.now();
+  const operationName = 'payRequest';
+  
   try {
     const invoice = parsePaymentRequest({ request });
     if (!invoice) return false;
@@ -36,10 +39,13 @@ const payRequest = async ({ request, amount }: { request: string, amount: number
     // We need to set a max fee amount
     const maxFee = amount * parseFloat(maxRoutingFee);
 
+    // Use configurable pathfinding timeout, default to 60 seconds
+    const pathfindingTimeout = parseInt(process.env.LN_PATHFINDING_TIMEOUT || '60000');
+    
     const params : PayViaPaymentRequestParams = {
       lnd,
       request,
-      pathfinding_timeout: 60000,
+      pathfinding_timeout: pathfindingTimeout,
     };
     // If the invoice doesn't have amount we add it to the params
     if (!invoice.tokens) params.tokens = amount;
@@ -51,12 +57,36 @@ const payRequest = async ({ request, amount }: { request: string, amount: number
     // Delete all routing reputations to clear pathfinding memory
     await deleteForwardingReputations({ lnd });
 
+    logger.info(`Starting payment for ${amount} sats with ${pathfindingTimeout}ms timeout`);
     const payment = await payViaPaymentRequest(params);
-
+    
+    logOperationDuration(operationName, startTime, true);
     return payment;
-  } catch (error) {
-    logger.error(`payRequest: ${error}`);
-    return false;
+  } catch (error: any) {
+    const errorMessage = error.toString();
+    const pathfindingTimeout = parseInt(process.env.LN_PATHFINDING_TIMEOUT || '60000');
+    
+    logOperationDuration(operationName, startTime, false);
+    
+    // Enhanced error handling for different timeout scenarios
+    if (errorMessage.includes('TimeoutError') || errorMessage.includes('timed out')) {
+      logTimeout('payRequest', pathfindingTimeout, error);
+      logger.error(`payRequest timeout after ${pathfindingTimeout}ms: ${errorMessage}`);
+      return { error: 'TIMEOUT', message: errorMessage };
+    }
+    
+    if (errorMessage.includes('UnknownPaymentHash') || errorMessage.includes('PaymentPathfindingFailedToFindPossibleRoute')) {
+      logger.error(`payRequest routing failed: ${errorMessage}`);
+      return { error: 'ROUTING_FAILED', message: errorMessage };
+    }
+    
+    if (errorMessage.includes('InsufficientBalance')) {
+      logger.error(`payRequest insufficient balance: ${errorMessage}`);
+      return { error: 'INSUFFICIENT_BALANCE', message: errorMessage };
+    }
+    
+    logger.error(`payRequest unexpected error: ${errorMessage}`);
+    return { error: 'UNKNOWN', message: errorMessage };
   }
 };
 
@@ -104,14 +134,34 @@ const payToBuyer = async (bot: HasTelegram, order: IOrder) => {
       );
       await messages.rateUserMessage(bot, buyerUser, order, i18nCtx);
     } else {
-      await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+      // Handle different types of payment failures
+      if (payment && typeof payment === 'object' && 'error' in payment) {
+        const errorType = payment.error as string;
+        
+        if (errorType === 'TIMEOUT') {
+          logger.warn(`Payment timeout for order ${order._id}, will retry later`);
+          await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+        } else if (errorType === 'ROUTING_FAILED') {
+          logger.warn(`Routing failed for order ${order._id}, will retry with cleared reputation`);
+          await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+        } else {
+          logger.error(`Payment failed for order ${order._id} with error: ${errorType}`);
+          await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+        }
+      } else {
+        await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+      }
+      
+      // Create pending payment for retry
       const pp = new PendingPayment({
         amount: order.amount,
         payment_request: order.buyer_invoice,
         user_id: buyerUser._id,
         description: order.description,
         hash: order.hash,
         order_id: order._id,
+        attempts: 1,
+        last_error: payment && typeof payment === 'object' && 'error' in payment ? payment.error as string : 'UNKNOWN',
       });
       await pp.save();
     }

logger.ts

@@ -23,5 +23,31 @@ const logger = winston.createLogger({
   exitOnError: false,
 });
 
+// Enhanced timeout monitoring utilities
+export const logTimeout = (operation: string, timeout: number, error?: any) => {
+  const logData = {
+    operation,
+    timeout_ms: timeout,
+    timestamp: new Date().toISOString(),
+    error: error?.toString() || 'Unknown timeout error'
+  };
+  logger.error(`TIMEOUT_MONITOR: ${JSON.stringify(logData)}`);
+};
+
+export const logOperationDuration = (operation: string, startTime: number, success: boolean = true) => {
+  const duration = Date.now() - startTime;
+  const logData = {
+    operation,
+    duration_ms: duration,
+    success,
+    timestamp: new Date().toISOString()
+  };
+  
+  if (duration > 30000) { // Log slow operations (>30s)
+    logger.warn(`SLOW_OPERATION: ${JSON.stringify(logData)}`);
+  } else {
+    logger.info(`OPERATION_DURATION: ${JSON.stringify(logData)}`);
+  }
+};
 
 export { logger };

models/pending_payment.ts

@@ -13,6 +13,8 @@ export interface IPendingPayment extends Document {
     user_id: string;
     order_id: string;
     community_id: string;
+    last_error: string;
+    next_retry: Date;
 }
 
 
@@ -37,6 +39,8 @@ const PendingPaymentSchema = new Schema<IPendingPayment>({
     user_id: { type: String },
     order_id: { type: String },
     community_id: { type: String },
+    last_error: { type: String, default: '' },
+    next_retry: { type: Date, default: () => new Date(Date.now() + 5 * 60 * 1000) },
 });