diff --git a/.gitignore b/.gitignore index ce5ad76..20b46e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +/.idea +/.cache data/ logs/ configs/config.yaml diff --git a/configs/config.example.yaml b/configs/config.example.yaml index 81fa271..a334d9f 100644 --- a/configs/config.example.yaml +++ b/configs/config.example.yaml @@ -91,6 +91,27 @@ chains: - url: "https://bsc.blockrazor.xyz" - url: "https://bnb.rpc.subquery.network/public" + conflux_mainnet: + network_id: "conflux_mainnet" + internal_code: "CONFLUX_MAINNET" + type: "evm" + start_block: 0 + poll_interval: "3s" + nodes: + - url: "https://evm.confluxrpc.com" + - url: "https://conflux-espace-public.unifra.io" + - url: "https://1rpc.io/cfx" + + conflux_testnet: + network_id: "conflux_testnet" + internal_code: "CONFLUX_TESTNET" + type: "evm" + start_block: 0 + poll_interval: "3s" + nodes: + - url: "https://evmtestnet.confluxrpc.com" + - url: "https://conflux-espace-testnet.public.blastapi.io" + bitcoin_testnet: network_id: "bitcoin_testnet" internal_code: "BTC_TESTNET" @@ -332,6 +353,86 @@ chains: batch_size: 20 concurrency: 4 + xrp_mainnet: + network_id: "xrp_mainnet" + internal_code: "XRP_MAINNET" + type: "xrp" + start_block: 0 + poll_interval: "2s" + nodes: + - url: "https://xrplcluster.com" + - url: "https://s1.ripple.com:51234" + client: + timeout: "15s" + max_retries: 3 + retry_delay: "1s" + throttle: + rps: 40 + burst: 80 + batch_size: 20 + concurrency: 6 + parallel: true + + xrp_testnet: + network_id: "xrp_testnet" + internal_code: "XRP_TESTNET" + type: "xrp" + start_block: 0 + poll_interval: "2s" + nodes: + - url: "https://s.altnet.rippletest.net:51234" + - url: "https://testnet.xrpl-labs.com" + client: + timeout: "15s" + max_retries: 3 + retry_delay: "1s" + throttle: + rps: 40 + burst: 80 + batch_size: 20 + concurrency: 6 + parallel: true + + stellar_mainnet: + network_id: "stellar_mainnet" + internal_code: "STELLAR_MAINNET" + type: "stellar" + start_block: 0 + poll_interval: "2s" + nodes: + - url: "https://horizon.stellar.org" + - url: "https://horizon.publicnode.com" + client: + timeout: "15s" + max_retries: 3 + retry_delay: "1s" + throttle: + rps: 40 + burst: 80 + batch_size: 20 + concurrency: 6 + parallel: true + + stellar_testnet: + network_id: "stellar_testnet" + internal_code: "STELLAR_TESTNET" + type: "stellar" + start_block: 0 + poll_interval: "2s" + nodes: + - url: "https://horizon-testnet.stellar.org" + - url: "https://horizon-testnet.publicnode.com" + client: + timeout: "15s" + max_retries: 3 + retry_delay: "1s" + throttle: + rps: 40 + burst: 80 + batch_size: 20 + concurrency: 6 + parallel: true + ton_mainnet: network_id: "ton" internal_code: "TON_MAINNET" diff --git a/internal/indexer/directional.go b/internal/indexer/directional.go new file mode 100644 index 0000000..ec27dce --- /dev/null +++ b/internal/indexer/directional.go @@ -0,0 +1,54 @@ +package indexer + +import ( + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/types" +) + +const ( + metadataKeySourceAmount = "source_amount" + metadataKeySourceAsset = "source_asset_address" + metadataKeySourceTxType = "source_tx_type" +) + +func normalizeDirectionalMetadata(tx types.Transaction, direction string) types.Transaction { + if direction != types.DirectionOut { + clearDirectionalMetadata(&tx) + return tx + } + + sourceType := tx.GetMetadataString(metadataKeySourceTxType) + if sourceType != "" { + tx.Type = constant.TxType(sourceType) + if tx.Type == constant.TxTypeNativeTransfer { + tx.AssetAddress = "" + } + } + + if sourceAmount := tx.GetMetadataString(metadataKeySourceAmount); sourceAmount != "" { + tx.Amount = sourceAmount + } + + switch tx.GetMetadataString(metadataKeySourceTxType) { + case string(constant.TxTypeNativeTransfer): + tx.AssetAddress = "" + case string(constant.TxTypeTokenTransfer): + tx.AssetAddress = tx.GetMetadataString(metadataKeySourceAsset) + } + + clearDirectionalMetadata(&tx) + return tx +} + +func clearDirectionalMetadata(tx *types.Transaction) { + if tx == nil || tx.Metadata == nil { + return + } + + delete(tx.Metadata, metadataKeySourceAmount) + delete(tx.Metadata, metadataKeySourceAsset) + delete(tx.Metadata, metadataKeySourceTxType) + if len(tx.Metadata) == 0 { + tx.Metadata = nil + } +} diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 664611b..9a41dc2 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -19,3 +19,11 @@ type Indexer interface { GetBlocksByNumbers(ctx context.Context, blockNumbers []uint64) ([]BlockResult, error) IsHealthy() bool } + +// DirectionalNormalizer is an optional hook for chains whose outgoing wallet +// view differs from the incoming payload of the same routed transfer +// (for example, path/cross-asset payments). It must not change from/to-based +// routing decisions; it only shapes the emitted transaction payload. +type DirectionalNormalizer interface { + NormalizeForDirection(tx types.Transaction, direction string) types.Transaction +} diff --git a/internal/indexer/stellar.go b/internal/indexer/stellar.go new file mode 100644 index 0000000..577611a --- /dev/null +++ b/internal/indexer/stellar.go @@ -0,0 +1,1013 @@ +package indexer + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/fystack/multichain-indexer/internal/rpc" + "github.com/fystack/multichain-indexer/internal/rpc/stellar" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/enum" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/fystack/multichain-indexer/pkg/infra" + "github.com/shopspring/decimal" +) + +const ( + stellarPaymentsPageLimit = 200 + stellarOperationsPageLimit = 200 + stellarEffectsPageLimit = 200 + stellarBurnAddress = "stellar:burn" + stellarClaimableBalanceStateScope = "stellar_claimable_balance" +) + +type StellarIndexer struct { + chainName string + config config.ChainConfig + failover *rpc.Failover[stellar.StellarAPI] + kvstore infra.KVStore + pubkeyStore PubkeyStore +} + +func NewStellarIndexer( + chainName string, + cfg config.ChainConfig, + failover *rpc.Failover[stellar.StellarAPI], + kvstore infra.KVStore, + pubkeyStore PubkeyStore, +) *StellarIndexer { + return &StellarIndexer{ + chainName: chainName, + config: cfg, + failover: failover, + kvstore: kvstore, + pubkeyStore: pubkeyStore, + } +} + +func (s *StellarIndexer) GetName() string { return strings.ToUpper(s.chainName) } +func (s *StellarIndexer) GetNetworkType() enum.NetworkType { return enum.NetworkTypeStellar } +func (s *StellarIndexer) GetNetworkInternalCode() string { return s.config.InternalCode } +func (s *StellarIndexer) NormalizeForDirection(tx types.Transaction, direction string) types.Transaction { + return normalizeDirectionalMetadata(tx, direction) +} + +func (s *StellarIndexer) isMonitoredTransfer(from, to string) bool { + if s.pubkeyStore == nil { + return true + } + + to = normalizeStellarAddress(to) + if to != "" && s.pubkeyStore.Exist(enum.NetworkTypeStellar, to) { + return true + } + + if !s.config.TwoWayIndexing { + return false + } + + from = normalizeStellarAddress(from) + return from != "" && s.pubkeyStore.Exist(enum.NetworkTypeStellar, from) +} + +func (s *StellarIndexer) GetLatestBlockNumber(ctx context.Context) (uint64, error) { + var latest uint64 + err := s.failover.ExecuteWithRetry(ctx, func(client stellar.StellarAPI) error { + n, err := client.GetLatestLedgerSequence(ctx) + latest = n + return err + }) + return latest, err +} + +func (s *StellarIndexer) GetBlock(ctx context.Context, number uint64) (*types.Block, error) { + var ( + ledger *stellar.Ledger + payments []stellar.Payment + operations []stellar.Operation + ) + err := s.failover.ExecuteWithRetry(ctx, func(client stellar.StellarAPI) error { + l, err := client.GetLedger(ctx, number) + if err != nil { + return err + } + ledger = l + + allPayments, err := fetchStellarLedgerPayments(ctx, client, number) + if err != nil { + return err + } + payments = allPayments + + allOperations, err := fetchStellarLedgerOperations(ctx, client, number) + if err != nil { + return err + } + operations = allOperations + return nil + }) + if err != nil { + return nil, fmt.Errorf("get stellar ledger %d failed: %w", number, err) + } + if ledger == nil { + return nil, fmt.Errorf("stellar ledger %d not found", number) + } + return s.convertLedger(ctx, ledger, payments, operations) +} + +func (s *StellarIndexer) GetBlocks(ctx context.Context, from, to uint64, isParallel bool) ([]BlockResult, error) { + if to < from { + return nil, fmt.Errorf("invalid range: from %d > to %d", from, to) + } + blockNumbers := make([]uint64, 0, to-from+1) + for n := from; n <= to; n++ { + blockNumbers = append(blockNumbers, n) + } + return s.GetBlocksByNumbers(ctx, blockNumbers) +} + +func (s *StellarIndexer) GetBlocksByNumbers(ctx context.Context, blockNumbers []uint64) ([]BlockResult, error) { + if len(blockNumbers) == 0 { + return nil, nil + } + + workers := s.config.Throttle.Concurrency + if workers <= 0 { + workers = 1 + } + workers = min(workers, len(blockNumbers)) + + results := make([]BlockResult, len(blockNumbers)) + type job struct { + index int + num uint64 + } + jobs := make(chan job, workers*2) + var wg sync.WaitGroup + + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := range jobs { + block, err := s.GetBlock(ctx, j.num) + results[j.index] = BlockResult{Number: j.num, Block: block} + if err != nil { + results[j.index].Error = &Error{ + ErrorType: classifyStellarError(err), + Message: err.Error(), + } + } + } + }() + } + + go func() { + defer close(jobs) + for i, num := range blockNumbers { + select { + case <-ctx.Done(): + return + case jobs <- job{index: i, num: num}: + } + } + }() + + wg.Wait() + if ctx.Err() != nil { + return nil, ctx.Err() + } + return results, firstBlockError(results) +} + +func (s *StellarIndexer) IsHealthy() bool { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := s.GetLatestBlockNumber(ctx) + return err == nil +} + +func (s *StellarIndexer) convertLedger( + ctx context.Context, + ledger *stellar.Ledger, + payments []stellar.Payment, + operations []stellar.Operation, +) (*types.Block, error) { + if ledger == nil { + return nil, fmt.Errorf("stellar ledger is nil") + } + + timestamp, err := parseRFC3339Unix(ledger.ClosedAt) + if err != nil { + return nil, fmt.Errorf("parse stellar ledger time: %w", err) + } + + txs := make([]types.Transaction, 0, len(payments)+len(operations)) + blockHash := strings.TrimSpace(ledger.Hash) + txDetails := make(map[string]*stellar.Transaction) + txDetailsFetched := make(map[string]bool) + effectsByOperation := make(map[string][]stellar.Effect) + effectsFetched := make(map[string]bool) + for paymentIndex, payment := range payments { + if !s.shouldProcessPayment(payment) { + continue + } + txDetail, err := s.getCachedTransactionDetail(ctx, txDetails, txDetailsFetched, payment.TransactionHash) + if err != nil { + return nil, err + } + converted, ok := s.convertPayment( + payment, + txDetail, + ledger.Sequence, + blockHash, + stellarTransferIndex(payment, paymentIndex), + timestamp, + ) + if !ok { + continue + } + txs = append(txs, converted) + } + for operationIndex, operation := range operations { + if !s.shouldProcessOperation(operation) { + continue + } + + var effects []stellar.Effect + if stellarOperationNeedsEffects(operation.Type) { + effects, err = s.getCachedOperationEffects(ctx, effectsByOperation, effectsFetched, operation.ID) + if err != nil { + return nil, err + } + } + + txDetail, err := s.getCachedTransactionDetail(ctx, txDetails, txDetailsFetched, operation.TransactionHash) + if err != nil { + return nil, err + } + converted, ok, err := s.convertOperation( + operation, + effects, + txDetail, + ledger.Sequence, + blockHash, + stellarOperationTransferIndex(operation, operationIndex), + timestamp, + ) + if err != nil { + return nil, err + } + if !ok { + continue + } + txs = append(txs, converted) + } + + return &types.Block{ + Number: ledger.Sequence, + Hash: ledger.Hash, + ParentHash: ledger.PrevHash, + Timestamp: timestamp, + Transactions: txs, + }, nil +} + +func (s *StellarIndexer) shouldProcessPayment(payment stellar.Payment) bool { + if !payment.TransactionSuccessful { + return false + } + if !isSupportedStellarPayment(payment.Type) { + return false + } + + from, to, amount := stellarTransferFields(payment) + if from == "" || to == "" || amount == "" { + return false + } + if !s.isMonitoredTransfer(from, to) { + return false + } + if isNativeStellarPayment(payment) { + return true + } + return formatStellarAsset(payment.AssetIssuer, payment.AssetCode) != "" +} + +func (s *StellarIndexer) shouldProcessOperation(operation stellar.Operation) bool { + if !operation.TransactionSuccessful { + return false + } + + switch strings.ToLower(strings.TrimSpace(operation.Type)) { + case "clawback": + from := normalizeStellarAddress(operation.From) + if from == "" || !s.isMonitoredAddress(from) { + return false + } + txType, _, ok := stellarAssetFromOperation(operation.Asset) + return ok && txType == constant.TxTypeTokenTransfer && strings.TrimSpace(operation.Amount) != "" + case "create_claimable_balance", "claim_claimable_balance", "clawback_claimable_balance": + return true + default: + return false + } +} + +func stellarOperationNeedsEffects(operationType string) bool { + switch strings.ToLower(strings.TrimSpace(operationType)) { + case "create_claimable_balance", "claim_claimable_balance": + return true + default: + return false + } +} + +func (s *StellarIndexer) getCachedTransactionDetail( + ctx context.Context, + cache map[string]*stellar.Transaction, + fetched map[string]bool, + hash string, +) (*stellar.Transaction, error) { + hash = strings.TrimSpace(hash) + if hash == "" { + return nil, nil + } + if fetched[hash] { + return cache[hash], nil + } + + var txDetail *stellar.Transaction + err := s.failover.ExecuteWithRetry(ctx, func(client stellar.StellarAPI) error { + tx, err := client.GetTransaction(ctx, hash) + txDetail = tx + return err + }) + if err != nil { + return nil, fmt.Errorf("get stellar transaction %s failed: %w", hash, err) + } + + cache[hash] = txDetail + fetched[hash] = true + return txDetail, nil +} + +func (s *StellarIndexer) getCachedOperationEffects( + ctx context.Context, + cache map[string][]stellar.Effect, + fetched map[string]bool, + operationID string, +) ([]stellar.Effect, error) { + operationID = strings.TrimSpace(operationID) + if operationID == "" { + return nil, nil + } + if fetched[operationID] { + return cache[operationID], nil + } + + var effects []stellar.Effect + err := s.failover.ExecuteWithRetry(ctx, func(client stellar.StellarAPI) error { + var err error + effects, err = fetchStellarOperationEffects(ctx, client, operationID) + return err + }) + if err != nil { + return nil, fmt.Errorf("get stellar effects for operation %s failed: %w", operationID, err) + } + + cache[operationID] = effects + fetched[operationID] = true + return effects, nil +} + +func (s *StellarIndexer) convertPayment( + payment stellar.Payment, + txDetail *stellar.Transaction, + ledgerSequence uint64, + blockHash string, + transferIndex string, + ledgerTimestamp uint64, +) (types.Transaction, bool) { + if !payment.TransactionSuccessful { + return types.Transaction{}, false + } + if !isSupportedStellarPayment(payment.Type) { + return types.Transaction{}, false + } + + from, to, amount := stellarTransferFields(payment) + if from == "" || to == "" || amount == "" { + return types.Transaction{}, false + } + if !s.isMonitoredTransfer(from, to) { + return types.Transaction{}, false + } + + assetAddress := "" + txType := constant.TxTypeNativeTransfer + if !isNativeStellarPayment(payment) { + txType = constant.TxTypeTokenTransfer + assetAddress = formatStellarAsset(payment.AssetIssuer, payment.AssetCode) + if assetAddress == "" { + return types.Transaction{}, false + } + } + + timestamp := ledgerTimestamp + if txDetail != nil && strings.TrimSpace(txDetail.CreatedAt) != "" { + parsed, err := parseRFC3339Unix(txDetail.CreatedAt) + if err == nil { + timestamp = parsed + } + } + + fee := decimal.Zero + memo := "" + var memoType types.MemoType + if txDetail != nil { + fee = stroopsToXLM(strings.TrimSpace(txDetail.FeeCharged)) + memo = strings.TrimSpace(txDetail.Memo) + memoType = types.MemoType(strings.TrimSpace(txDetail.MemoType)) + } + + tx := types.Transaction{ + TxHash: strings.TrimSpace(payment.TransactionHash), + NetworkId: s.config.NetworkId, + BlockNumber: ledgerSequence, + BlockHash: blockHash, + TransferIndex: transferIndex, + FromAddress: normalizeStellarAddress(from), + ToAddress: normalizeStellarAddress(to), + AssetAddress: assetAddress, + Amount: amount, + Type: txType, + TxFee: fee, + Timestamp: timestamp, + Confirmations: 1, + Status: types.StatusConfirmed, + } + tx.Memo = memo + tx.MemoType = memoType + if sourceType, sourceAssetAddress, sourceAmount, ok := stellarSourcePaymentDetails(payment); ok { + tx.SetMetadata(metadataKeySourceTxType, string(sourceType)) + tx.SetMetadata(metadataKeySourceAmount, sourceAmount) + if sourceType == constant.TxTypeTokenTransfer { + tx.SetMetadata(metadataKeySourceAsset, sourceAssetAddress) + } + } + + return tx, true +} + +type stellarClaimableBalanceState struct { + Asset string `json:"asset"` + Amount string `json:"amount"` + SourceAccount string `json:"source_account"` + Claimants []string `json:"claimants"` +} + +func (s *StellarIndexer) convertOperation( + operation stellar.Operation, + effects []stellar.Effect, + txDetail *stellar.Transaction, + ledgerSequence uint64, + blockHash string, + transferIndex string, + ledgerTimestamp uint64, +) (types.Transaction, bool, error) { + if !operation.TransactionSuccessful { + return types.Transaction{}, false, nil + } + + switch strings.ToLower(strings.TrimSpace(operation.Type)) { + case "clawback": + tx, ok := s.convertClawbackOperation(operation, txDetail, ledgerSequence, blockHash, transferIndex, ledgerTimestamp) + return tx, ok, nil + case "create_claimable_balance": + return s.convertCreateClaimableBalanceOperation(operation, effects, txDetail, ledgerSequence, blockHash, transferIndex, ledgerTimestamp) + case "claim_claimable_balance": + return s.convertClaimClaimableBalanceOperation(operation, effects, txDetail, ledgerSequence, blockHash, transferIndex, ledgerTimestamp) + case "clawback_claimable_balance": + return s.convertClawbackClaimableBalanceOperation(operation) + default: + return types.Transaction{}, false, nil + } +} + +func (s *StellarIndexer) convertClawbackOperation( + operation stellar.Operation, + txDetail *stellar.Transaction, + ledgerSequence uint64, + blockHash string, + transferIndex string, + ledgerTimestamp uint64, +) (types.Transaction, bool) { + from := normalizeStellarAddress(operation.From) + if from == "" || !s.isMonitoredAddress(from) { + return types.Transaction{}, false + } + + txType, assetAddress, ok := stellarAssetFromOperation(operation.Asset) + if !ok || txType != constant.TxTypeTokenTransfer || strings.TrimSpace(operation.Amount) == "" { + return types.Transaction{}, false + } + + tx := s.newOperationTransaction(operation, txDetail, ledgerSequence, blockHash, transferIndex, ledgerTimestamp) + tx.FromAddress = from + tx.ToAddress = stellarBurnAddress + tx.AssetAddress = assetAddress + tx.Amount = strings.TrimSpace(operation.Amount) + tx.Type = txType + tx.SetMetadata(types.MetadataKeySubtype, "clawback") + return tx, true +} + +func (s *StellarIndexer) convertCreateClaimableBalanceOperation( + operation stellar.Operation, + effects []stellar.Effect, + txDetail *stellar.Transaction, + ledgerSequence uint64, + blockHash string, + transferIndex string, + ledgerTimestamp uint64, +) (types.Transaction, bool, error) { + if s.kvstore == nil { + return types.Transaction{}, false, fmt.Errorf("stellar claimable balance state requires kvstore") + } + + effect := findStellarEffect(effects, "claimable_balance_created") + balanceID := strings.TrimSpace(operation.BalanceID) + if balanceID == "" && effect != nil { + balanceID = strings.TrimSpace(effect.BalanceID) + } + if balanceID == "" { + return types.Transaction{}, false, nil + } + + asset := strings.TrimSpace(operation.Asset) + if asset == "" && effect != nil { + asset = strings.TrimSpace(effect.Asset) + } + amount := strings.TrimSpace(operation.Amount) + if amount == "" && effect != nil { + amount = strings.TrimSpace(effect.Amount) + } + txType, assetAddress, ok := stellarAssetFromOperation(asset) + if !ok || amount == "" { + return types.Transaction{}, false, nil + } + + sourceAccount := normalizeStellarAddress(operation.SourceAccount) + claimants := stellarClaimantAddresses(operation.Claimants) + state := stellarClaimableBalanceState{ + Asset: asset, + Amount: amount, + SourceAccount: sourceAccount, + Claimants: claimants, + } + if err := s.saveClaimableBalanceState(balanceID, state); err != nil { + return types.Transaction{}, false, err + } + + shouldEmit := s.config.TwoWayIndexing && s.isMonitoredAddress(sourceAccount) + if !shouldEmit { + return types.Transaction{}, false, nil + } + + tx := s.newOperationTransaction(operation, txDetail, ledgerSequence, blockHash, transferIndex, ledgerTimestamp) + tx.FromAddress = sourceAccount + tx.AssetAddress = assetAddress + tx.Amount = amount + tx.Type = txType + tx.SetMetadata(types.MetadataKeySubtype, "create_claimable_balance") + tx.SetMetadata(types.MetadataKeyClaimableID, balanceID) + if len(claimants) > 0 { + tx.SetMetadata(types.MetadataKeyClaimants, claimants) + } + return tx, true, nil +} + +func (s *StellarIndexer) convertClaimClaimableBalanceOperation( + operation stellar.Operation, + effects []stellar.Effect, + txDetail *stellar.Transaction, + ledgerSequence uint64, + blockHash string, + transferIndex string, + ledgerTimestamp uint64, +) (types.Transaction, bool, error) { + balanceID := strings.TrimSpace(operation.BalanceID) + if balanceID == "" { + return types.Transaction{}, false, nil + } + state, found, err := s.loadClaimableBalanceState(balanceID) + if err != nil { + return types.Transaction{}, false, err + } + if found { + if err := s.deleteClaimableBalanceState(balanceID); err != nil { + return types.Transaction{}, false, err + } + } else { + state, found = stellarClaimableBalanceStateFromEffect(findStellarEffect(effects, "claimable_balance_claimed")) + if !found { + return types.Transaction{}, false, nil + } + } + + claimant := normalizeStellarAddress(operation.Claimant) + if claimant == "" || !s.isMonitoredAddress(claimant) { + return types.Transaction{}, false, nil + } + + txType, assetAddress, ok := stellarAssetFromOperation(state.Asset) + if !ok || strings.TrimSpace(state.Amount) == "" { + return types.Transaction{}, false, nil + } + + tx := s.newOperationTransaction(operation, txDetail, ledgerSequence, blockHash, transferIndex, ledgerTimestamp) + tx.ToAddress = claimant + tx.AssetAddress = assetAddress + tx.Amount = strings.TrimSpace(state.Amount) + tx.Type = txType + tx.SetMetadata(types.MetadataKeySubtype, "claim_claimable_balance") + tx.SetMetadata(types.MetadataKeyClaimableID, balanceID) + return tx, true, nil +} + +func (s *StellarIndexer) convertClawbackClaimableBalanceOperation( + operation stellar.Operation) (types.Transaction, bool, error) { + balanceID := strings.TrimSpace(operation.BalanceID) + if balanceID == "" { + return types.Transaction{}, false, nil + } + _, found, err := s.loadClaimableBalanceState(balanceID) + if err != nil { + return types.Transaction{}, false, err + } + if found { + if err := s.deleteClaimableBalanceState(balanceID); err != nil { + return types.Transaction{}, false, err + } + } + return types.Transaction{}, false, nil +} + +func (s *StellarIndexer) newOperationTransaction( + operation stellar.Operation, + txDetail *stellar.Transaction, + ledgerSequence uint64, + blockHash string, + transferIndex string, + ledgerTimestamp uint64, +) types.Transaction { + timestamp := ledgerTimestamp + if txDetail != nil && strings.TrimSpace(txDetail.CreatedAt) != "" { + parsed, err := parseRFC3339Unix(txDetail.CreatedAt) + if err == nil { + timestamp = parsed + } + } else if strings.TrimSpace(operation.CreatedAt) != "" { + parsed, err := parseRFC3339Unix(operation.CreatedAt) + if err == nil { + timestamp = parsed + } + } + + fee := decimal.Zero + if txDetail != nil { + fee = stroopsToXLM(strings.TrimSpace(txDetail.FeeCharged)) + } + + tx := types.Transaction{ + TxHash: strings.TrimSpace(operation.TransactionHash), + NetworkId: s.config.NetworkId, + BlockNumber: ledgerSequence, + BlockHash: blockHash, + TransferIndex: transferIndex, + TxFee: fee, + Timestamp: timestamp, + Confirmations: 1, + Status: types.StatusConfirmed, + } + if txDetail != nil { + if memo := strings.TrimSpace(txDetail.Memo); memo != "" { + tx.Memo = memo + } + if memoType := strings.TrimSpace(txDetail.MemoType); memoType != "" { + tx.MemoType = types.MemoType(memoType) + } + } + return tx +} + +func (s *StellarIndexer) saveClaimableBalanceState(balanceID string, state stellarClaimableBalanceState) error { + if s.kvstore == nil { + return fmt.Errorf("stellar claimable balance state requires kvstore") + } + return s.kvstore.SetAny(s.claimableBalanceStateKey(balanceID), state) +} + +func (s *StellarIndexer) loadClaimableBalanceState(balanceID string) (stellarClaimableBalanceState, bool, error) { + if s.kvstore == nil { + return stellarClaimableBalanceState{}, false, fmt.Errorf("stellar claimable balance state requires kvstore") + } + var state stellarClaimableBalanceState + found, err := s.kvstore.GetAny(s.claimableBalanceStateKey(balanceID), &state) + return state, found, err +} + +func (s *StellarIndexer) deleteClaimableBalanceState(balanceID string) error { + if s.kvstore == nil { + return fmt.Errorf("stellar claimable balance state requires kvstore") + } + return s.kvstore.Delete(s.claimableBalanceStateKey(balanceID)) +} + +func (s *StellarIndexer) claimableBalanceStateKey(balanceID string) string { + return fmt.Sprintf("%s/%s/%s", stellarClaimableBalanceStateScope, strings.TrimSpace(s.config.NetworkId), strings.TrimSpace(balanceID)) +} + +func (s *StellarIndexer) isMonitoredAddress(address string) bool { + if s.pubkeyStore == nil { + return true + } + address = normalizeStellarAddress(address) + return address != "" && s.pubkeyStore.Exist(enum.NetworkTypeStellar, address) +} + +func stellarAssetFromOperation(asset string) (constant.TxType, string, bool) { + asset = strings.TrimSpace(asset) + if asset == "" { + return "", "", false + } + if strings.EqualFold(asset, "native") { + return constant.TxTypeNativeTransfer, "", true + } + parts := strings.SplitN(asset, ":", 2) + if len(parts) != 2 { + return "", "", false + } + assetAddress := formatStellarAsset(parts[1], parts[0]) + if assetAddress == "" { + return "", "", false + } + return constant.TxTypeTokenTransfer, assetAddress, true +} + +func stellarClaimantAddresses(claimants []stellar.Claimant) []string { + addresses := make([]string, 0, len(claimants)) + for _, claimant := range claimants { + address := normalizeStellarAddress(claimant.Destination) + if address == "" { + continue + } + addresses = append(addresses, address) + } + return addresses +} + +func findStellarEffect(effects []stellar.Effect, effectType string) *stellar.Effect { + for i := range effects { + if strings.EqualFold(strings.TrimSpace(effects[i].Type), effectType) { + return &effects[i] + } + } + return nil +} + +func stellarClaimableBalanceStateFromEffect(effect *stellar.Effect) (stellarClaimableBalanceState, bool) { + if effect == nil { + return stellarClaimableBalanceState{}, false + } + + state := stellarClaimableBalanceState{ + Asset: strings.TrimSpace(effect.Asset), + Amount: strings.TrimSpace(effect.Amount), + } + if claimant := normalizeStellarAddress(effect.Account); claimant != "" { + state.Claimants = []string{claimant} + } + if state.Asset == "" || state.Amount == "" { + return stellarClaimableBalanceState{}, false + } + return state, true +} + +func stellarTransferIndex(payment stellar.Payment, paymentIndex int) string { + if pagingToken := strings.TrimSpace(payment.PagingToken); pagingToken != "" { + return pagingToken + } + if id := strings.TrimSpace(payment.ID); id != "" { + return id + } + return fmt.Sprintf("%d", paymentIndex) +} + +func stellarOperationTransferIndex(operation stellar.Operation, operationIndex int) string { + if pagingToken := strings.TrimSpace(operation.PagingToken); pagingToken != "" { + return pagingToken + } + if id := strings.TrimSpace(operation.ID); id != "" { + return id + } + return fmt.Sprintf("%d", operationIndex) +} + +func fetchStellarLedgerPayments(ctx context.Context, client stellar.StellarAPI, sequence uint64) ([]stellar.Payment, error) { + cursor := "" + payments := make([]stellar.Payment, 0, stellarPaymentsPageLimit) + + for { + prevCursor := cursor + page, err := client.GetPaymentsByLedger(ctx, sequence, cursor, stellarPaymentsPageLimit) + if err != nil { + return nil, err + } + if page == nil || len(page.Embedded.Records) == 0 { + break + } + + for _, payment := range page.Embedded.Records { + payments = append(payments, payment) + cursor = strings.TrimSpace(payment.PagingToken) + } + if cursor == "" || cursor == prevCursor { + break + } + } + + return payments, nil +} + +func fetchStellarLedgerOperations(ctx context.Context, client stellar.StellarAPI, sequence uint64) ([]stellar.Operation, error) { + cursor := "" + operations := make([]stellar.Operation, 0, stellarOperationsPageLimit) + + for { + prevCursor := cursor + page, err := client.GetOperationsByLedger(ctx, sequence, cursor, stellarOperationsPageLimit) + if err != nil { + return nil, err + } + if page == nil || len(page.Embedded.Records) == 0 { + break + } + + for _, operation := range page.Embedded.Records { + operations = append(operations, operation) + cursor = strings.TrimSpace(operation.PagingToken) + } + if cursor == "" || cursor == prevCursor { + break + } + } + + return operations, nil +} + +func fetchStellarOperationEffects(ctx context.Context, client stellar.StellarAPI, operationID string) ([]stellar.Effect, error) { + cursor := "" + effects := make([]stellar.Effect, 0, stellarEffectsPageLimit) + + for { + prevCursor := cursor + page, err := client.GetEffectsByOperation(ctx, operationID, cursor, stellarEffectsPageLimit) + if err != nil { + return nil, err + } + if page == nil || len(page.Embedded.Records) == 0 { + break + } + + for _, effect := range page.Embedded.Records { + effects = append(effects, effect) + cursor = strings.TrimSpace(effect.PagingToken) + } + if cursor == "" || cursor == prevCursor { + break + } + } + + return effects, nil +} + +func classifyStellarError(err error) ErrorType { + if err == nil { + return ErrorTypeUnknown + } + msg := strings.ToLower(err.Error()) + switch { + case strings.Contains(msg, "404"), strings.Contains(msg, "not found"): + return ErrorTypeBlockNotFound + case strings.Contains(msg, "timeout"): + return ErrorTypeTimeout + default: + return ErrorTypeUnknown + } +} + +func isSupportedStellarPayment(paymentType string) bool { + switch strings.ToLower(strings.TrimSpace(paymentType)) { + case "payment": + return true + case "create_account": + return true + case "path_payment_strict_receive": + return true + case "path_payment_strict_recieve": + return true + case "path_payment_strict_send": + return true + case "account_merge": + return true + default: + return false + } +} + +func isNativeStellarPayment(payment stellar.Payment) bool { + paymentType := strings.ToLower(strings.TrimSpace(payment.Type)) + if paymentType == "create_account" || paymentType == "account_merge" { + return true + } + return strings.EqualFold(strings.TrimSpace(payment.AssetType), "native") +} + +func stellarSourcePaymentDetails(payment stellar.Payment) (constant.TxType, string, string, bool) { + switch strings.ToLower(strings.TrimSpace(payment.Type)) { + case "path_payment_strict_receive", "path_payment_strict_recieve", "path_payment_strict_send": + default: + return "", "", "", false + } + + amount := strings.TrimSpace(payment.SourceAmount) + if amount == "" { + return "", "", "", false + } + if strings.EqualFold(strings.TrimSpace(payment.SourceAssetType), "native") { + return constant.TxTypeNativeTransfer, "", amount, true + } + + assetAddress := formatStellarAsset(payment.SourceAssetIssuer, payment.SourceAssetCode) + if assetAddress == "" { + return "", "", "", false + } + return constant.TxTypeTokenTransfer, assetAddress, amount, true +} + +func stellarTransferFields(payment stellar.Payment) (string, string, string) { + switch strings.ToLower(strings.TrimSpace(payment.Type)) { + case "create_account": + from := strings.TrimSpace(payment.Funder) + if from == "" { + from = strings.TrimSpace(payment.SourceAccount) + } + to := strings.TrimSpace(payment.Account) + if to == "" { + to = strings.TrimSpace(payment.Into) + } + return from, to, strings.TrimSpace(payment.StartingBalance) + case "account_merge": + from := strings.TrimSpace(payment.Account) + if from == "" { + from = strings.TrimSpace(payment.SourceAccount) + } + return from, strings.TrimSpace(payment.Into), strings.TrimSpace(payment.Amount) + default: + return strings.TrimSpace(payment.From), strings.TrimSpace(payment.To), strings.TrimSpace(payment.Amount) + } +} + +func formatStellarAsset(issuer string, code string) string { + issuer = normalizeStellarAddress(issuer) + code = strings.TrimSpace(code) + if issuer == "" || code == "" { + return "" + } + return issuer + ":" + code +} + +func normalizeStellarAddress(address string) string { + return strings.TrimSpace(address) +} + +func stroopsToXLM(v string) decimal.Decimal { + if strings.TrimSpace(v) == "" { + return decimal.Zero + } + return decimal.RequireFromString(strings.TrimSpace(v)).Div(decimal.NewFromInt(10_000_000)) +} + +func parseRFC3339Unix(v string) (uint64, error) { + ts, err := time.Parse(time.RFC3339, strings.TrimSpace(v)) + if err != nil { + return 0, err + } + return uint64(ts.Unix()), nil +} diff --git a/internal/indexer/stellar_test.go b/internal/indexer/stellar_test.go new file mode 100644 index 0000000..52abd5c --- /dev/null +++ b/internal/indexer/stellar_test.go @@ -0,0 +1,1353 @@ +package indexer + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "github.com/fystack/multichain-indexer/internal/rpc" + "github.com/fystack/multichain-indexer/internal/rpc/stellar" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/enum" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/fystack/multichain-indexer/pkg/infra" + "github.com/hashicorp/consul/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type mockStellarPubkeyStore struct { + addresses map[string]struct{} +} + +func (m mockStellarPubkeyStore) Exist(_ enum.NetworkType, address string) bool { + _, ok := m.addresses[address] + return ok +} + +type mockStellarAPI struct { + mu sync.Mutex + latest uint64 + ledgers map[uint64]*stellar.Ledger + pages map[uint64]map[string]*stellar.PaymentsPage + operations map[uint64]map[string]*stellar.OperationsPage + effects map[string]map[string]*stellar.EffectsPage + transactions map[string]*stellar.Transaction + txCalls map[string]int + effectCalls map[string]int +} + +var _ stellar.StellarAPI = (*mockStellarAPI)(nil) + +func (m *mockStellarAPI) CallRPC(context.Context, string, any) (*rpc.RPCResponse, error) { + return nil, nil +} +func (m *mockStellarAPI) Do(context.Context, string, string, any, map[string]string) ([]byte, error) { + return nil, nil +} +func (m *mockStellarAPI) GetNetworkType() string { return rpc.NetworkStellar } +func (m *mockStellarAPI) GetClientType() string { return rpc.ClientTypeREST } +func (m *mockStellarAPI) GetURL() string { return "https://stellar.test" } +func (m *mockStellarAPI) Close() error { return nil } + +func (m *mockStellarAPI) GetLatestLedgerSequence(context.Context) (uint64, error) { + return m.latest, nil +} +func (m *mockStellarAPI) GetLedger(_ context.Context, sequence uint64) (*stellar.Ledger, error) { + return m.ledgers[sequence], nil +} +func (m *mockStellarAPI) GetPaymentsByLedger(_ context.Context, sequence uint64, cursor string, limit int) (*stellar.PaymentsPage, error) { + if pageSet, ok := m.pages[sequence]; ok { + return pageSet[cursor], nil + } + return nil, nil +} +func (m *mockStellarAPI) GetOperationsByLedger(_ context.Context, sequence uint64, cursor string, limit int) (*stellar.OperationsPage, error) { + if pageSet, ok := m.operations[sequence]; ok { + return pageSet[cursor], nil + } + return nil, nil +} +func (m *mockStellarAPI) GetEffectsByOperation(_ context.Context, operationID string, cursor string, limit int) (*stellar.EffectsPage, error) { + m.mu.Lock() + if m.effectCalls != nil { + m.effectCalls[operationID]++ + } + m.mu.Unlock() + if pageSet, ok := m.effects[operationID]; ok { + return pageSet[cursor], nil + } + return nil, nil +} +func (m *mockStellarAPI) GetTransaction(_ context.Context, hash string) (*stellar.Transaction, error) { + m.mu.Lock() + if m.txCalls != nil { + m.txCalls[hash]++ + } + m.mu.Unlock() + return m.transactions[hash], nil +} + +const stellarMainnetHorizon = "https://horizon.stellar.org" + +func newTestStellarLiveIndexer(t *testing.T, kvstore infra.KVStore) (*StellarIndexer, *stellar.Client) { + t.Helper() + + client := stellar.NewClient(stellarMainnetHorizon, nil, 30*time.Second, nil) + t.Cleanup(func() { + _ = client.Close() + }) + + failover := rpc.NewFailover[stellar.StellarAPI](nil) + require.NoError(t, failover.AddProvider(&rpc.Provider{ + Name: "stellar-mainnet", + URL: stellarMainnetHorizon, + Network: "stellar_mainnet", + ClientType: rpc.ClientTypeREST, + Client: client, + State: rpc.StateHealthy, + })) + + return NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet", TwoWayIndexing: true}, + failover, + kvstore, + nil, + ), client +} + +func findStellarPayment(t *testing.T, payments []stellar.Payment, pagingToken string) stellar.Payment { + t.Helper() + + for _, payment := range payments { + if payment.PagingToken == pagingToken { + return payment + } + } + + t.Fatalf("payment with paging token %s not found", pagingToken) + return stellar.Payment{} +} + +func findStellarOperation(t *testing.T, operations []stellar.Operation, operationID string) stellar.Operation { + t.Helper() + + for _, operation := range operations { + if operation.ID == operationID { + return operation + } + } + + t.Fatalf("operation %s not found", operationID) + return stellar.Operation{} +} + +type mockKVStore struct { + mu sync.RWMutex + items map[string][]byte +} + +func newMockKVStore() *mockKVStore { + return &mockKVStore{items: make(map[string][]byte)} +} + +func (m *mockKVStore) GetName() string { return "mock" } + +func (m *mockKVStore) Set(k string, v string) error { + m.mu.Lock() + defer m.mu.Unlock() + m.items[k] = []byte(v) + return nil +} + +func (m *mockKVStore) Get(k string) (string, error) { + m.mu.RLock() + defer m.mu.RUnlock() + val, ok := m.items[k] + if !ok { + return "", nil + } + return string(val), nil +} + +func (m *mockKVStore) GetWithOptions(k string, _ *api.QueryOptions) (string, error) { + return m.Get(k) +} + +func (m *mockKVStore) SetAny(k string, v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + m.items[k] = data + return nil +} + +func (m *mockKVStore) GetAny(k string, v any) (bool, error) { + m.mu.RLock() + data, ok := m.items[k] + m.mu.RUnlock() + if !ok { + return false, nil + } + return true, json.Unmarshal(data, v) +} + +func (m *mockKVStore) List(prefix string) ([]*infra.KVPair, error) { + m.mu.RLock() + defer m.mu.RUnlock() + result := make([]*infra.KVPair, 0) + for key, value := range m.items { + if len(prefix) == 0 || key == prefix || (len(key) > len(prefix) && key[:len(prefix)] == prefix) { + result = append(result, &infra.KVPair{Key: key, Value: value}) + } + } + return result, nil +} + +func (m *mockKVStore) Delete(k string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.items, k) + return nil +} + +func (m *mockKVStore) BatchSet(pairs []infra.KVPair) error { + m.mu.Lock() + defer m.mu.Unlock() + for _, pair := range pairs { + m.items[pair.Key] = pair.Value + } + return nil +} + +func (m *mockKVStore) Close() error { return nil } + +func TestStellarGetBlock_ParsesNativeAndIssuedPaymentsWithMemo(t *testing.T) { + t.Parallel() + + api := &mockStellarAPI{ + latest: 15, + ledgers: map[uint64]*stellar.Ledger{ + 10: { + Hash: "LEDGER_HASH", + PrevHash: "PARENT_HASH", + Sequence: 10, + ClosedAt: "2026-03-18T10:00:00Z", + }, + }, + pages: map[uint64]map[string]*stellar.PaymentsPage{ + 10: { + "": { + Embedded: struct { + Records []stellar.Payment `json:"records"` + }{ + Records: []stellar.Payment{ + { + ID: "1", + PagingToken: "pt1", + Type: "payment", + TransactionHash: "tx-native", + TransactionSuccessful: true, + From: "GFROM", + To: "GDEST", + Amount: "12.5000000", + AssetType: "native", + CreatedAt: "2026-03-18T10:00:00Z", + }, + { + ID: "2", + PagingToken: "pt2", + Type: "payment", + TransactionHash: "tx-token", + TransactionSuccessful: true, + From: "GISSUERFROM", + To: "GDEST", + Amount: "99.2500000", + AssetType: "credit_alphanum4", + AssetCode: "USDC", + AssetIssuer: "GISSUER", + CreatedAt: "2026-03-18T10:00:02Z", + }, + { + ID: "3", + PagingToken: "pt3", + Type: "create_account", + TransactionHash: "tx-create", + TransactionSuccessful: true, + Funder: "GCREATOR", + Account: "GDEST", + StartingBalance: "10000.0000000", + CreatedAt: "2026-03-18T10:00:03Z", + }, + { + ID: "4", + PagingToken: "pt4", + Type: "path_payment_strict_send", + TransactionHash: "tx-path", + TransactionSuccessful: true, + From: "GPATHFROM", + To: "GDEST", + Amount: "7.1250000", + AssetType: "native", + CreatedAt: "2026-03-18T10:00:04Z", + }, + }, + }, + }, + }, + }, + transactions: map[string]*stellar.Transaction{ + "tx-native": { + Hash: "tx-native", + Successful: true, + FeeCharged: "100", + Memo: "memo-1", + MemoType: "text", + Ledger: 10, + CreatedAt: "2026-03-18T10:00:00Z", + }, + "tx-token": { + Hash: "tx-token", + Successful: true, + FeeCharged: "200", + Memo: "memo-2", + MemoType: "text", + Ledger: 10, + CreatedAt: "2026-03-18T10:00:02Z", + }, + "tx-create": { + Hash: "tx-create", + Successful: true, + FeeCharged: "100", + Memo: "", + MemoType: "none", + Ledger: 10, + CreatedAt: "2026-03-18T10:00:03Z", + }, + "tx-path": { + Hash: "tx-path", + Successful: true, + FeeCharged: "300", + Memo: "memo-3", + MemoType: "text", + Ledger: 10, + CreatedAt: "2026-03-18T10:00:04Z", + }, + }, + } + + failover := rpc.NewFailover[stellar.StellarAPI](nil) + require.NoError(t, failover.AddProvider(&rpc.Provider{ + Name: "stellar-test", + URL: "https://stellar.test", + Network: "stellar_mainnet", + ClientType: rpc.ClientTypeREST, + Client: api, + State: rpc.StateHealthy, + })) + + idx := NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet"}, + failover, + nil, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GDEST": {}}}, + ) + + block, err := idx.GetBlock(context.Background(), 10) + require.NoError(t, err) + require.Len(t, block.Transactions, 4) + + nativeTx := block.Transactions[0] + assert.Equal(t, constant.TxTypeNativeTransfer, nativeTx.Type) + assert.Equal(t, "", nativeTx.AssetAddress) + assert.Equal(t, "LEDGER_HASH", nativeTx.BlockHash) + assert.Equal(t, "pt1", nativeTx.TransferIndex) + memo := nativeTx.Memo + assert.Equal(t, "memo-1", memo) + memoType := nativeTx.MemoType + assert.Equal(t, types.MemoTypeText, memoType) + assert.Equal(t, "0.00001", nativeTx.TxFee.String()) + + tokenTx := block.Transactions[1] + assert.Equal(t, constant.TxTypeTokenTransfer, tokenTx.Type) + assert.Equal(t, "GISSUER:USDC", tokenTx.AssetAddress) + assert.Equal(t, "LEDGER_HASH", tokenTx.BlockHash) + assert.Equal(t, "pt2", tokenTx.TransferIndex) + memo = tokenTx.Memo + assert.Equal(t, "memo-2", memo) + assert.Equal(t, "99.2500000", tokenTx.Amount) + + createTx := block.Transactions[2] + assert.Equal(t, constant.TxTypeNativeTransfer, createTx.Type) + assert.Equal(t, "LEDGER_HASH", createTx.BlockHash) + assert.Equal(t, "pt3", createTx.TransferIndex) + assert.Equal(t, "GCREATOR", createTx.FromAddress) + assert.Equal(t, "GDEST", createTx.ToAddress) + assert.Equal(t, "10000.0000000", createTx.Amount) + + pathTx := block.Transactions[3] + assert.Equal(t, constant.TxTypeNativeTransfer, pathTx.Type) + assert.Equal(t, "LEDGER_HASH", pathTx.BlockHash) + assert.Equal(t, "pt4", pathTx.TransferIndex) + assert.Equal(t, "GPATHFROM", pathTx.FromAddress) + assert.Equal(t, "GDEST", pathTx.ToAddress) + assert.Equal(t, "7.1250000", pathTx.Amount) + memo = pathTx.Memo + assert.Equal(t, "memo-3", memo) + assert.Equal(t, "0.00003", pathTx.TxFee.String()) +} + +func TestStellarConvertPayment_PathPaymentStoresSourceSideRouting(t *testing.T) { + t.Parallel() + + idx := NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet"}, + nil, + nil, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GDEST": {}}}, + ) + + tx, ok := idx.convertPayment( + stellar.Payment{ + Type: "path_payment_strict_send", + TransactionHash: "tx-path", + TransactionSuccessful: true, + From: "GSOURCE", + To: "GDEST", + Amount: "5.0000000", + AssetType: "credit_alphanum4", + AssetCode: "USDC", + AssetIssuer: "GISSUER", + SourceAmount: "10.0000000", + SourceAssetType: "native", + }, + nil, + 21, + "ledger-hash", + "payment-1", + 123, + ) + require.True(t, ok) + assert.Equal(t, constant.TxTypeTokenTransfer, tx.Type) + assert.Equal(t, "GISSUER:USDC", tx.AssetAddress) + assert.Equal(t, "5.0000000", tx.Amount) + assert.Equal(t, string(constant.TxTypeNativeTransfer), tx.GetMetadataString(metadataKeySourceTxType)) + assert.Equal(t, "10.0000000", tx.GetMetadataString(metadataKeySourceAmount)) + assert.Equal(t, "", tx.GetMetadataString(metadataKeySourceAsset)) + + outTx := idx.NormalizeForDirection(tx, types.DirectionOut) + assert.Equal(t, constant.TxTypeNativeTransfer, outTx.Type) + assert.Equal(t, "", outTx.AssetAddress) + assert.Equal(t, "10.0000000", outTx.Amount) + assert.Equal(t, "", outTx.GetMetadataString(metadataKeySourceTxType)) + assert.Equal(t, "", outTx.GetMetadataString(metadataKeySourceAmount)) + assert.Equal(t, "", outTx.GetMetadataString(metadataKeySourceAsset)) +} + +func TestStellarGetBlock_FetchesTransactionDetailsLazily(t *testing.T) { + t.Parallel() + + api := &mockStellarAPI{ + latest: 20, + ledgers: map[uint64]*stellar.Ledger{ + 20: { + Hash: "LEDGER_HASH", + PrevHash: "PARENT_HASH", + Sequence: 20, + ClosedAt: "2026-03-18T11:00:00Z", + }, + }, + pages: map[uint64]map[string]*stellar.PaymentsPage{ + 20: { + "": { + Embedded: struct { + Records []stellar.Payment `json:"records"` + }{ + Records: []stellar.Payment{ + { + ID: "1", + PagingToken: "pt1", + Type: "payment", + TransactionHash: "tx-pay", + TransactionSuccessful: true, + From: "GFROM", + To: "GDEST", + Amount: "1.0000000", + AssetType: "native", + }, + { + ID: "2", + PagingToken: "pt2", + Type: "payment", + TransactionHash: "tx-pay", + TransactionSuccessful: true, + From: "GFROM2", + To: "GDEST", + Amount: "2.0000000", + AssetType: "native", + }, + }, + }, + }, + }, + }, + operations: map[uint64]map[string]*stellar.OperationsPage{ + 20: { + "": { + Embedded: struct { + Records []stellar.Operation `json:"records"` + }{ + Records: []stellar.Operation{ + { + ID: "op-ignored", + PagingToken: "op-ignored", + Type: "manage_sell_offer", + TransactionHash: "tx-ignored", + TransactionSuccessful: true, + }, + { + ID: "op-create", + PagingToken: "op-create", + Type: "create_claimable_balance", + TransactionHash: "tx-create", + TransactionSuccessful: true, + SourceAccount: "GSOURCE", + Asset: "USD:GISSUER", + Amount: "5.5000000", + Claimants: []stellar.Claimant{ + {Destination: "GCLAIMANT"}, + }, + }, + { + ID: "op-claim", + PagingToken: "op-claim", + Type: "claim_claimable_balance", + TransactionHash: "tx-claim", + TransactionSuccessful: true, + BalanceID: "balance-2", + Claimant: "GCLAIMANT", + }, + { + ID: "op-clawback", + PagingToken: "op-clawback", + Type: "clawback_claimable_balance", + TransactionHash: "tx-clawback", + TransactionSuccessful: true, + BalanceID: "balance-3", + }, + }, + }, + }, + }, + }, + effects: map[string]map[string]*stellar.EffectsPage{ + "op-create": { + "": { + Embedded: struct { + Records []stellar.Effect `json:"records"` + }{ + Records: []stellar.Effect{ + { + Type: "claimable_balance_created", + BalanceID: "balance-1", + Asset: "USD:GISSUER", + Amount: "5.5000000", + }, + }, + }, + }, + }, + "op-claim": { + "": { + Embedded: struct { + Records []stellar.Effect `json:"records"` + }{ + Records: []stellar.Effect{ + { + Type: "claimable_balance_claimed", + Account: "GCLAIMANT", + BalanceID: "balance-2", + Asset: "USD:GISSUER", + Amount: "4.2500000", + }, + }, + }, + }, + }, + "op-clawback": { + "": { + Embedded: struct { + Records []stellar.Effect `json:"records"` + }{ + Records: []stellar.Effect{ + { + Type: "claimable_balance_clawed_back", + Account: "GCLAIMANT", + BalanceID: "balance-3", + Asset: "USD:GISSUER", + Amount: "1.7500000", + }, + }, + }, + }, + }, + }, + transactions: map[string]*stellar.Transaction{ + "tx-pay": { + Hash: "tx-pay", + Successful: true, + FeeCharged: "100", + Ledger: 20, + CreatedAt: "2026-03-18T11:00:01Z", + }, + "tx-create": { + Hash: "tx-create", + Successful: true, + FeeCharged: "200", + Ledger: 20, + CreatedAt: "2026-03-18T11:00:02Z", + }, + "tx-claim": { + Hash: "tx-claim", + Successful: true, + FeeCharged: "210", + Ledger: 20, + CreatedAt: "2026-03-18T11:00:02Z", + }, + "tx-clawback": { + Hash: "tx-clawback", + Successful: true, + FeeCharged: "220", + Ledger: 20, + CreatedAt: "2026-03-18T11:00:02Z", + }, + "tx-ignored": { + Hash: "tx-ignored", + Successful: true, + FeeCharged: "300", + Ledger: 20, + CreatedAt: "2026-03-18T11:00:03Z", + }, + }, + txCalls: map[string]int{}, + effectCalls: map[string]int{}, + } + + failover := rpc.NewFailover[stellar.StellarAPI](nil) + require.NoError(t, failover.AddProvider(&rpc.Provider{ + Name: "stellar-test", + URL: "https://stellar.test", + Network: "stellar_mainnet", + ClientType: rpc.ClientTypeREST, + Client: api, + State: rpc.StateHealthy, + })) + + idx := NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet"}, + failover, + newMockKVStore(), + mockStellarPubkeyStore{addresses: map[string]struct{}{ + "GDEST": {}, + "GCLAIMANT": {}, + }}, + ) + + block, err := idx.GetBlock(context.Background(), 20) + require.NoError(t, err) + require.NotNil(t, block) + require.Len(t, block.Transactions, 3) + + assert.Equal(t, 1, api.txCalls["tx-pay"]) + assert.Equal(t, 1, api.txCalls["tx-create"]) + assert.Equal(t, 1, api.txCalls["tx-claim"]) + assert.Equal(t, 1, api.txCalls["tx-clawback"]) + assert.Zero(t, api.txCalls["tx-ignored"]) + assert.Equal(t, 1, api.effectCalls["op-create"]) + assert.Equal(t, 1, api.effectCalls["op-claim"]) + assert.Zero(t, api.effectCalls["op-clawback"]) + + claimTx := block.Transactions[2] + assert.Empty(t, claimTx.FromAddress) + assert.Equal(t, "GCLAIMANT", claimTx.ToAddress) + assert.Equal(t, "GISSUER:USD", claimTx.AssetAddress) + assert.Equal(t, "4.2500000", claimTx.Amount) + assert.Equal(t, "claim_claimable_balance", claimTx.GetMetadataString(types.MetadataKeySubtype)) +} + +func TestFetchStellarLedgerPayments_PaginatesWithoutDuplication(t *testing.T) { + t.Parallel() + + api := &mockStellarAPI{ + pages: map[uint64]map[string]*stellar.PaymentsPage{ + 11: { + "": { + Embedded: struct { + Records []stellar.Payment `json:"records"` + }{ + Records: []stellar.Payment{ + {PagingToken: "p1", TransactionHash: "tx1"}, + {PagingToken: "p2", TransactionHash: "tx2"}, + }, + }, + }, + "p2": { + Embedded: struct { + Records []stellar.Payment `json:"records"` + }{ + Records: []stellar.Payment{ + {PagingToken: "p3", TransactionHash: "tx3"}, + }, + }, + }, + }, + }, + } + + payments, err := fetchStellarLedgerPayments(context.Background(), api, 11) + require.NoError(t, err) + require.Len(t, payments, 3) + assert.Equal(t, "tx1", payments[0].TransactionHash) + assert.Equal(t, "tx2", payments[1].TransactionHash) + assert.Equal(t, "tx3", payments[2].TransactionHash) +} + +func TestStellarConvertPayment_RespectsTwoWayIndexing(t *testing.T) { + t.Parallel() + + payment := stellar.Payment{ + Type: "payment", + TransactionHash: "tx-out", + TransactionSuccessful: true, + From: "GMONITORED", + To: "GOTHER", + Amount: "1.0000000", + AssetType: "native", + } + + idx := NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet"}, + nil, + nil, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GMONITORED": {}}}, + ) + _, ok := idx.convertPayment(payment, nil, 12, "ledger-hash", "payment-0", 1) + require.False(t, ok) + + idx = NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet", TwoWayIndexing: true}, + nil, + nil, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GMONITORED": {}}}, + ) + _, ok = idx.convertPayment(payment, nil, 12, "ledger-hash", "payment-0", 1) + require.True(t, ok) +} + +func TestStellarConvertPayment_CreateAccountUsesFunderAndStartingBalance(t *testing.T) { + t.Parallel() + + payment := stellar.Payment{ + Type: "create_account", + TransactionHash: "tx-create", + TransactionSuccessful: true, + Funder: "GFUNDER", + Account: "GNEW", + StartingBalance: "10000.0000000", + } + + idx := NewStellarIndexer( + "stellar_testnet", + config.ChainConfig{NetworkId: "stellar_testnet"}, + nil, + nil, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GNEW": {}}}, + ) + + tx, ok := idx.convertPayment(payment, nil, 42, "ledger-hash", "payment-1", 123) + require.True(t, ok) + assert.Equal(t, "ledger-hash", tx.BlockHash) + assert.Equal(t, "payment-1", tx.TransferIndex) + assert.Equal(t, "GFUNDER", tx.FromAddress) + assert.Equal(t, "GNEW", tx.ToAddress) + assert.Equal(t, "10000.0000000", tx.Amount) + assert.Equal(t, constant.TxTypeNativeTransfer, tx.Type) +} + +func TestStellarConvertPayment_AccountMergeUsesIntoAndAmount(t *testing.T) { + t.Parallel() + + payment := stellar.Payment{ + Type: "account_merge", + TransactionHash: "tx-merge", + TransactionSuccessful: true, + Account: "GMERGED", + Into: "GDEST", + Amount: "8.5000000", + } + + idx := NewStellarIndexer( + "stellar_testnet", + config.ChainConfig{NetworkId: "stellar_testnet"}, + nil, + nil, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GDEST": {}}}, + ) + + tx, ok := idx.convertPayment(payment, nil, 52, "ledger-hash", "payment-2", 123) + require.True(t, ok) + assert.Equal(t, "ledger-hash", tx.BlockHash) + assert.Equal(t, "payment-2", tx.TransferIndex) + assert.Equal(t, "GMERGED", tx.FromAddress) + assert.Equal(t, "GDEST", tx.ToAddress) + assert.Equal(t, "8.5000000", tx.Amount) + assert.Equal(t, constant.TxTypeNativeTransfer, tx.Type) +} + +func TestStellarConvertOperation_ClawbackUsesBurnAddress(t *testing.T) { + t.Parallel() + + idx := NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet"}, + nil, + nil, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GHOLDER": {}}}, + ) + + tx, ok, err := idx.convertOperation( + stellar.Operation{ + ID: "10", + PagingToken: "10", + Type: "clawback", + TransactionHash: "tx-claw", + TransactionSuccessful: true, + From: "GHOLDER", + Asset: "USDC:GISSUER", + Amount: "5.0000000", + CreatedAt: "2026-03-18T10:00:00Z", + }, + nil, + &stellar.Transaction{FeeCharged: "100"}, + 77, + "ledger-hash", + "10", + 123, + ) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "GHOLDER", tx.FromAddress) + assert.Equal(t, stellarBurnAddress, tx.ToAddress) + assert.Equal(t, "GISSUER:USDC", tx.AssetAddress) + assert.Equal(t, "5.0000000", tx.Amount) + assert.Equal(t, constant.TxTypeTokenTransfer, tx.Type) + assert.Equal(t, "ledger-hash", tx.BlockHash) + assert.Equal(t, "10", tx.TransferIndex) + subtype := tx.GetMetadataString(types.MetadataKeySubtype) + assert.Equal(t, "clawback", subtype) +} + +func TestStellarConvertOperation_CreateClaimableBalancePersistsState(t *testing.T) { + t.Parallel() + + kv := newMockKVStore() + idx := NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet"}, + nil, + kv, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GCLAIM": {}}}, + ) + + tx, ok, err := idx.convertOperation( + stellar.Operation{ + ID: "200", + PagingToken: "200", + Type: "create_claimable_balance", + TransactionHash: "tx-create-balance", + TransactionSuccessful: true, + SourceAccount: "GFUNDER", + Asset: "USDC:GISSUER", + Amount: "11.5000000", + Claimants: []stellar.Claimant{{Destination: "GCLAIM"}}, + CreatedAt: "2026-03-18T10:00:00Z", + }, + []stellar.Effect{ + {Type: "claimable_balance_created", BalanceID: "balance-1"}, + }, + &stellar.Transaction{FeeCharged: "100"}, + 77, + "ledger-hash", + "200", + 123, + ) + require.NoError(t, err) + assert.False(t, ok) + assert.Empty(t, tx.FromAddress) + assert.Empty(t, tx.ToAddress) + + var state stellarClaimableBalanceState + found, err := kv.GetAny(idx.claimableBalanceStateKey("balance-1"), &state) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, "USDC:GISSUER", state.Asset) + assert.Equal(t, "11.5000000", state.Amount) + assert.Equal(t, "GFUNDER", state.SourceAccount) + assert.Equal(t, []string{"GCLAIM"}, state.Claimants) +} + +func TestStellarConvertOperation_CreateClaimableBalanceEmitsSourceDebitWhenTwoWayIndexed(t *testing.T) { + t.Parallel() + + kv := newMockKVStore() + idx := NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet", TwoWayIndexing: true}, + nil, + kv, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GFUNDER": {}}}, + ) + + tx, ok, err := idx.convertOperation( + stellar.Operation{ + ID: "201", + PagingToken: "201", + Type: "create_claimable_balance", + TransactionHash: "tx-create-balance", + TransactionSuccessful: true, + SourceAccount: "GFUNDER", + Asset: "USDC:GISSUER", + Amount: "11.5000000", + Claimants: []stellar.Claimant{{Destination: "GCLAIM"}}, + }, + []stellar.Effect{ + {Type: "claimable_balance_created", BalanceID: "balance-1"}, + }, + &stellar.Transaction{FeeCharged: "100"}, + 77, + "ledger-hash", + "201", + 123, + ) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "GFUNDER", tx.FromAddress) + assert.Empty(t, tx.ToAddress) + assert.Equal(t, "GISSUER:USDC", tx.AssetAddress) + assert.Equal(t, "11.5000000", tx.Amount) + assert.Equal(t, constant.TxTypeTokenTransfer, tx.Type) + assert.Equal(t, "create_claimable_balance", tx.GetMetadataString(types.MetadataKeySubtype)) + assert.Equal(t, "balance-1", tx.GetMetadataString(types.MetadataKeyClaimableID)) +} + +func TestStellarConvertOperation_ClaimClaimableBalanceConsumesState(t *testing.T) { + t.Parallel() + + kv := newMockKVStore() + idx := NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet"}, + nil, + kv, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GCLAIM": {}}}, + ) + require.NoError(t, idx.saveClaimableBalanceState("balance-2", stellarClaimableBalanceState{ + Asset: "USDC:GISSUER", + Amount: "7.0000000", + SourceAccount: "GFUNDER", + Claimants: []string{"GCLAIM"}, + })) + + tx, ok, err := idx.convertOperation( + stellar.Operation{ + ID: "201", + PagingToken: "201", + Type: "claim_claimable_balance", + TransactionHash: "tx-claim-balance", + TransactionSuccessful: true, + BalanceID: "balance-2", + Claimant: "GCLAIM", + }, + nil, + &stellar.Transaction{FeeCharged: "100"}, + 78, + "ledger-hash", + "201", + 123, + ) + require.NoError(t, err) + require.True(t, ok) + assert.Empty(t, tx.FromAddress) + assert.Equal(t, "GCLAIM", tx.ToAddress) + assert.Equal(t, "GISSUER:USDC", tx.AssetAddress) + assert.Equal(t, "7.0000000", tx.Amount) + assert.Equal(t, constant.TxTypeTokenTransfer, tx.Type) + + var state stellarClaimableBalanceState + found, err := kv.GetAny(idx.claimableBalanceStateKey("balance-2"), &state) + require.NoError(t, err) + assert.False(t, found) +} + +func TestStellarConvertOperation_ClawbackClaimableBalanceConsumesState(t *testing.T) { + t.Parallel() + + kv := newMockKVStore() + idx := NewStellarIndexer( + "stellar_mainnet", + config.ChainConfig{NetworkId: "stellar_mainnet"}, + nil, + kv, + mockStellarPubkeyStore{addresses: map[string]struct{}{"GCLAIM": {}}}, + ) + require.NoError(t, idx.saveClaimableBalanceState("balance-3", stellarClaimableBalanceState{ + Asset: "USDC:GISSUER", + Amount: "3.5000000", + SourceAccount: "GFUNDER", + Claimants: []string{"GCLAIM"}, + })) + + tx, ok, err := idx.convertOperation( + stellar.Operation{ + ID: "202", + PagingToken: "202", + Type: "clawback_claimable_balance", + TransactionHash: "tx-clawback-balance", + TransactionSuccessful: true, + BalanceID: "balance-3", + }, + nil, + &stellar.Transaction{FeeCharged: "100"}, + 79, + "ledger-hash", + "202", + 123, + ) + require.NoError(t, err) + assert.False(t, ok) + assert.Empty(t, tx.FromAddress) + assert.Empty(t, tx.ToAddress) + + var state stellarClaimableBalanceState + found, err := kv.GetAny(idx.claimableBalanceStateKey("balance-3"), &state) + require.NoError(t, err) + assert.False(t, found) +} + +func TestStellarMainnetFetchAndParseTransactions(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + type stellarRealTxCase struct { + name string + kind string + txHash string + operationID string + transferIndex string + wantType constant.TxType + wantFrom string + wantTo string + wantAmount string + wantMetadata map[string]string + beforeConvert func(t *testing.T, idx *StellarIndexer, operation stellar.Operation, effects []stellar.Effect) + verify func(t *testing.T, tx types.Transaction, idx *StellarIndexer, payment *stellar.Payment, operation *stellar.Operation, effects []stellar.Effect) + } + + testCases := []stellarRealTxCase{ + { + name: "native payment", + kind: "payment", + txHash: "81dba977244285fd3a6e9192ff2a594fe99d9bd6ec892533fe80c5f25c77c1f0", + transferIndex: "265365546521460737", + wantType: constant.TxTypeNativeTransfer, + wantFrom: "GBC6NRTTQLRCABQHIR5J4R4YDJWFWRAO4ZRQIM2SVI5GSIZ2HZ42RINW", + wantTo: "GCZNOTQRRETQLBQH2MPWYMCLQBYMXKZI7XXYHS7F5RJHH7VMATQ57TQZ", + wantAmount: "120.0621870", + verify: func(t *testing.T, tx types.Transaction, _ *StellarIndexer, _ *stellar.Payment, _ *stellar.Operation, _ []stellar.Effect) { + assert.Equal(t, "", tx.AssetAddress) + }, + }, + { + name: "payment with memo", + kind: "payment", + txHash: "5701567d4bcabbf54b43a084c0fc0b9fc544358d6b2aa70603ab0e6a9a8c2f81", + transferIndex: "265854060396396545", + wantType: constant.TxTypeTokenTransfer, + wantFrom: "GBN32NH6TMWE4ZD4G245CF3UVOQRXD4FK3FDCLZ4DE5642HWFKSLLRMB", + wantTo: "GBKZQ2BYMOEJESIERVLWVSZD7CNNB7U2IMDCCWPDUN46AQAAXY3Y7I3C", + wantAmount: "18.8900000", + verify: func(t *testing.T, tx types.Transaction, _ *StellarIndexer, payment *stellar.Payment, _ *stellar.Operation, _ []stellar.Effect) { + require.NotNil(t, payment) + assert.Equal(t, formatStellarAsset(payment.AssetIssuer, payment.AssetCode), tx.AssetAddress) + assert.Equal(t, "pspb:4441859", tx.Memo) + assert.Equal(t, types.MemoTypeText, tx.MemoType) + }, + }, + { + name: "create account", + kind: "payment", + txHash: "981290ff3d4707fbb0413fd2e3aab41c359e19e626dcc3dfce5a88fc424937c6", + transferIndex: "265348783263514625", + wantType: constant.TxTypeNativeTransfer, + wantFrom: "GDNHPXSFIZQMJJFBAFWUWG3442AHGI3WEWUYRYXIGMJRHDUQOPHTKLDC", + wantTo: "GBMB4C3RRTYWQZNTRZGKD4Z4OLMVRJZPFCQUY6UIRUACG7FLCYHAEGPN", + wantAmount: "1.5000000", + }, + { + name: "payment", + kind: "payment", + txHash: "4003ae37c7b5126c364e6890a64792f57b268e1bebb43546e0347ba113d16530", + transferIndex: "265336753060651009", + wantType: constant.TxTypeTokenTransfer, + wantFrom: "GAMV73I2KDYWDLGO5SNVL5IA6JSGLCALD7XJSXHPUB27MMJNTDQIX3UC", + wantTo: "GBN32NH6TMWE4ZD4G245CF3UVOQRXD4FK3FDCLZ4DE5642HWFKSLLRMB", + wantAmount: "56.6500000", + verify: func(t *testing.T, tx types.Transaction, _ *StellarIndexer, payment *stellar.Payment, _ *stellar.Operation, _ []stellar.Effect) { + require.NotNil(t, payment) + assert.Equal(t, formatStellarAsset(payment.AssetIssuer, payment.AssetCode), tx.AssetAddress) + }, + }, + { + name: "path payment strict send", + kind: "payment", + txHash: "9eeb46b6a608dde3985d1dd9d4110f030a6902c7ea0586de0d51cf52ce055b2c", + transferIndex: "265348641529663489", + wantType: constant.TxTypeTokenTransfer, + wantFrom: "GBOA3PJZ3EUWLWLNXPICD57STTQSQKGEKDXNXC2VVPTML7SCQMAYUO3G", + wantTo: "GBOA3PJZ3EUWLWLNXPICD57STTQSQKGEKDXNXC2VVPTML7SCQMAYUO3G", + wantAmount: "1.9175151", + verify: func(t *testing.T, tx types.Transaction, _ *StellarIndexer, payment *stellar.Payment, _ *stellar.Operation, _ []stellar.Effect) { + require.NotNil(t, payment) + assert.Equal(t, "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN:USDC", tx.AssetAddress) + }, + }, + { + name: "create claimable balance", + kind: "operation", + txHash: "2308f9725aafde7aa6249e10abb1ccd3218116c6ee8d56305977db1cdc30319f", + operationID: "265336628506390529", + transferIndex: "265336628506390529", + wantType: constant.TxTypeTokenTransfer, + wantFrom: "GCKIK5UOKKGXCDCWGY3BAKJ2T5H6BXOXHI4SMMEUSUE6NWLJP3F2KQUU", + wantTo: "", + wantAmount: "0.0010000", + wantMetadata: map[string]string{ + types.MetadataKeySubtype: "create_claimable_balance", + types.MetadataKeyClaimableID: "000000009289e60febbca7863b48b8cf4d68b514d779ef13d1078b2fcd6d9371b0c3647a", + }, + verify: func(t *testing.T, tx types.Transaction, idx *StellarIndexer, _ *stellar.Payment, operation *stellar.Operation, _ []stellar.Effect) { + require.NotNil(t, operation) + assert.Equal(t, "GBZH36ATUXJZKFRMQTAAW42MWNM34SOA4N6E7DQ62V3G5NVITC3QOVRL:OVRL", tx.AssetAddress) + + state, found, err := idx.loadClaimableBalanceState("000000009289e60febbca7863b48b8cf4d68b514d779ef13d1078b2fcd6d9371b0c3647a") + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, operation.Asset, state.Asset) + assert.Equal(t, operation.Amount, state.Amount) + assert.Equal(t, operation.SourceAccount, state.SourceAccount) + assert.Equal(t, stellarClaimantAddresses(operation.Claimants), state.Claimants) + }, + }, + { + name: "claim claimable balance", + kind: "operation", + txHash: "eaef80df07081e46288947f6e286f25c09ff552ab861b5f90a2cdfbd1976e880", + operationID: "265334914814349334", + transferIndex: "265334914814349334", + wantType: constant.TxTypeTokenTransfer, + wantFrom: "", + wantTo: "GCKIK5UOKKGXCDCWGY3BAKJ2T5H6BXOXHI4SMMEUSUE6NWLJP3F2KQUU", + wantMetadata: map[string]string{ + types.MetadataKeySubtype: "claim_claimable_balance", + types.MetadataKeyClaimableID: "0000000022b1a16b0a8d5aa67f932dc7a4f28f0ae44b7b270af4fce16a02362eea7e3897", + }, + beforeConvert: func(t *testing.T, idx *StellarIndexer, operation stellar.Operation, effects []stellar.Effect) { + effect := findStellarEffect(effects, "claimable_balance_claimed") + require.NotNil(t, effect) + require.NoError(t, idx.saveClaimableBalanceState("0000000022b1a16b0a8d5aa67f932dc7a4f28f0ae44b7b270af4fce16a02362eea7e3897", stellarClaimableBalanceState{ + Asset: effect.Asset, + Amount: effect.Amount, + SourceAccount: operation.Claimant, + Claimants: []string{operation.Claimant}, + })) + }, + verify: func(t *testing.T, tx types.Transaction, idx *StellarIndexer, _ *stellar.Payment, _ *stellar.Operation, effects []stellar.Effect) { + effect := findStellarEffect(effects, "claimable_balance_claimed") + require.NotNil(t, effect) + assert.Equal(t, effect.Amount, tx.Amount) + assert.Equal(t, "GBZH36ATUXJZKFRMQTAAW42MWNM34SOA4N6E7DQ62V3G5NVITC3QOVRL:OVRL", tx.AssetAddress) + + _, found, err := idx.loadClaimableBalanceState("0000000022b1a16b0a8d5aa67f932dc7a4f28f0ae44b7b270af4fce16a02362eea7e3897") + require.NoError(t, err) + assert.False(t, found) + }, + }, + } + + enabled := false + for _, tc := range testCases { + if tc.txHash != "" && tc.transferIndex != "" { + enabled = true + break + } + } + if !enabled { + t.Skip("hardcoded real mainnet test cases are empty") + } + + for _, tc := range testCases { + tc := tc + if tc.txHash == "" || tc.transferIndex == "" { + continue + } + + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + kvstore := newMockKVStore() + idx, client := newTestStellarLiveIndexer(t, kvstore) + + txDetail, err := client.GetTransaction(ctx, tc.txHash) + require.NoError(t, err, "should fetch real mainnet transaction %s", tc.txHash) + require.NotNil(t, txDetail) + + ledger, err := client.GetLedger(ctx, txDetail.Ledger) + require.NoError(t, err, "should fetch real mainnet ledger %d", txDetail.Ledger) + require.NotNil(t, ledger) + + ledgerTimestamp, err := parseRFC3339Unix(ledger.ClosedAt) + require.NoError(t, err) + + var ( + tx types.Transaction + ok bool + payment *stellar.Payment + operation *stellar.Operation + effects []stellar.Effect + ) + + switch tc.kind { + case "payment": + payments, err := fetchStellarLedgerPayments(ctx, client, txDetail.Ledger) + require.NoError(t, err) + p := findStellarPayment(t, payments, tc.transferIndex) + payment = &p + tx, ok = idx.convertPayment(p, txDetail, ledger.Sequence, ledger.Hash, tc.transferIndex, ledgerTimestamp) + require.True(t, ok) + case "operation": + operations, err := fetchStellarLedgerOperations(ctx, client, txDetail.Ledger) + require.NoError(t, err) + op := findStellarOperation(t, operations, tc.operationID) + require.Equal(t, tc.transferIndex, op.PagingToken) + operation = &op + + effects, err = fetchStellarOperationEffects(ctx, client, tc.operationID) + require.NoError(t, err) + + if tc.beforeConvert != nil { + tc.beforeConvert(t, idx, op, effects) + } + + tx, ok, err = idx.convertOperation(op, effects, txDetail, ledger.Sequence, ledger.Hash, tc.transferIndex, ledgerTimestamp) + require.NoError(t, err) + require.True(t, ok) + default: + t.Fatalf("unsupported stellar mainnet test kind %q", tc.kind) + } + + require.NotEmpty(t, tx.TxHash) + require.NotEmpty(t, tx.Type) + + assert.Equal(t, tc.txHash, tx.TxHash) + assert.Equal(t, tc.wantFrom, tx.FromAddress) + assert.Equal(t, tc.wantTo, tx.ToAddress) + if tc.wantAmount != "" { + assert.Equal(t, tc.wantAmount, tx.Amount) + } + assert.Equal(t, tc.wantType, tx.Type) + assert.Equal(t, ledger.Hash, tx.BlockHash) + assert.Equal(t, tc.transferIndex, tx.TransferIndex) + + for key, want := range tc.wantMetadata { + assert.Equal(t, want, tx.GetMetadataString(key), "metadata %s", key) + } + + if tc.verify != nil { + tc.verify(t, tx, idx, payment, operation, effects) + } + }) + } +} + +func TestStellarMainnetBatchPaymentProducesMultipleTransactions(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + const batchTxHash = "5701567d4bcabbf54b43a084c0fc0b9fc544358d6b2aa70603ab0e6a9a8c2f81" + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + idx, client := newTestStellarLiveIndexer(t, newMockKVStore()) + + txDetail, err := client.GetTransaction(ctx, batchTxHash) + require.NoError(t, err) + require.NotNil(t, txDetail) + + ledger, err := client.GetLedger(ctx, txDetail.Ledger) + require.NoError(t, err) + require.NotNil(t, ledger) + + ledgerTimestamp, err := parseRFC3339Unix(ledger.ClosedAt) + require.NoError(t, err) + + payments, err := fetchStellarLedgerPayments(ctx, client, txDetail.Ledger) + require.NoError(t, err) + + converted := make([]types.Transaction, 0) + toSet := make(map[string]struct{}) + transferIndexSet := make(map[string]struct{}) + for _, payment := range payments { + if !strings.EqualFold(strings.TrimSpace(payment.TransactionHash), batchTxHash) { + continue + } + transferIndex := strings.TrimSpace(payment.PagingToken) + tx, ok := idx.convertPayment(payment, txDetail, ledger.Sequence, ledger.Hash, transferIndex, ledgerTimestamp) + if !ok { + continue + } + converted = append(converted, tx) + if tx.ToAddress != "" { + toSet[tx.ToAddress] = struct{}{} + } + if tx.TransferIndex != "" { + transferIndexSet[tx.TransferIndex] = struct{}{} + } + } + + require.Greater(t, len(converted), 1, "batch tx should emit multiple transfers") + require.Greater(t, len(toSet), 1, "batch tx should include multiple destination addresses") + assert.Len(t, transferIndexSet, len(converted), "each emitted transfer should have unique transferIndex") + + for _, tx := range converted { + assert.Equal(t, batchTxHash, tx.TxHash) + assert.Equal(t, "pspb:4441859", tx.Memo) + assert.Equal(t, types.MemoTypeText, tx.MemoType) + } +} diff --git a/internal/indexer/xrp.go b/internal/indexer/xrp.go new file mode 100644 index 0000000..4e7775d --- /dev/null +++ b/internal/indexer/xrp.go @@ -0,0 +1,980 @@ +package indexer + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/fystack/multichain-indexer/internal/rpc" + "github.com/fystack/multichain-indexer/internal/rpc/xrp" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/enum" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/shopspring/decimal" +) + +const ( + xrpEpochOffset = 946684800 + xrpBurnAddress = "xrp:burn" +) + +type XRPIndexer struct { + chainName string + config config.ChainConfig + failover *rpc.Failover[xrp.XRPLAPI] + pubkeyStore PubkeyStore +} + +func NewXRPIndexer( + chainName string, + cfg config.ChainConfig, + failover *rpc.Failover[xrp.XRPLAPI], + pubkeyStore PubkeyStore, +) *XRPIndexer { + return &XRPIndexer{ + chainName: chainName, + config: cfg, + failover: failover, + pubkeyStore: pubkeyStore, + } +} + +func (x *XRPIndexer) GetName() string { return strings.ToUpper(x.chainName) } +func (x *XRPIndexer) GetNetworkType() enum.NetworkType { return enum.NetworkTypeXRP } +func (x *XRPIndexer) GetNetworkInternalCode() string { return x.config.InternalCode } +func (x *XRPIndexer) NormalizeForDirection(tx types.Transaction, direction string) types.Transaction { + return normalizeDirectionalMetadata(tx, direction) +} + +func (x *XRPIndexer) isMonitoredTransfer(from, to string) bool { + if x.pubkeyStore == nil { + return true + } + + if candidate := normalizeXRPAddress(to); candidate != "" && x.pubkeyStore.Exist(enum.NetworkTypeXRP, candidate) { + return true + } + + if !x.config.TwoWayIndexing { + return false + } + + candidate := normalizeXRPAddress(from) + return candidate != "" && x.pubkeyStore.Exist(enum.NetworkTypeXRP, candidate) +} + +func (x *XRPIndexer) GetLatestBlockNumber(ctx context.Context) (uint64, error) { + var latest uint64 + err := x.failover.ExecuteWithRetry(ctx, func(client xrp.XRPLAPI) error { + n, err := client.GetLatestLedgerIndex(ctx) + latest = n + return err + }) + return latest, err +} + +func (x *XRPIndexer) GetBlock(ctx context.Context, number uint64) (*types.Block, error) { + var ledger *xrp.Ledger + err := x.failover.ExecuteWithRetry(ctx, func(client xrp.XRPLAPI) error { + l, err := client.GetLedgerByIndex(ctx, number) + ledger = l + return err + }) + if err != nil { + return nil, fmt.Errorf("get xrp ledger %d failed: %w", number, err) + } + if ledger == nil { + return nil, fmt.Errorf("xrp ledger %d not found", number) + } + return x.convertLedger(ledger, number) +} + +func (x *XRPIndexer) GetBlocks(ctx context.Context, from, to uint64, isParallel bool) ([]BlockResult, error) { + if to < from { + return nil, fmt.Errorf("invalid range: from %d > to %d", from, to) + } + blockNumbers := make([]uint64, 0, to-from+1) + for n := from; n <= to; n++ { + blockNumbers = append(blockNumbers, n) + } + return x.GetBlocksByNumbers(ctx, blockNumbers) +} + +func (x *XRPIndexer) GetBlocksByNumbers(ctx context.Context, blockNumbers []uint64) ([]BlockResult, error) { + if len(blockNumbers) == 0 { + return nil, nil + } + + results := make([]BlockResult, len(blockNumbers)) + indexByLedger := make(map[uint64]int, len(blockNumbers)) + for i, ledgerIndex := range blockNumbers { + indexByLedger[ledgerIndex] = i + } + + var ledgers map[uint64]*xrp.Ledger + err := x.failover.ExecuteWithRetry(ctx, func(client xrp.XRPLAPI) error { + var err error + ledgers, err = client.BatchGetLedgersByIndex(ctx, blockNumbers) + return err + }) + if err == nil { + for _, ledgerIndex := range blockNumbers { + ledger := ledgers[ledgerIndex] + idx := indexByLedger[ledgerIndex] + if ledger == nil { + results[idx] = BlockResult{ + Number: ledgerIndex, + Error: &Error{ErrorType: ErrorTypeBlockNotFound, Message: "ledger not found"}, + } + continue + } + block, convertErr := x.convertLedger(ledger, ledgerIndex) + results[idx] = BlockResult{Number: ledgerIndex, Block: block} + if convertErr != nil { + results[idx].Error = &Error{ErrorType: classifyXRPError(convertErr), Message: convertErr.Error()} + } + } + return results, firstBlockError(results) + } + + workers := x.config.Throttle.Concurrency + if workers <= 0 { + workers = 1 + } + workers = min(workers, len(blockNumbers)) + + type job struct { + index int + num uint64 + } + jobs := make(chan job, workers*2) + var wg sync.WaitGroup + + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := range jobs { + block, getErr := x.GetBlock(ctx, j.num) + results[j.index] = BlockResult{Number: j.num, Block: block} + if getErr != nil { + results[j.index].Error = &Error{ + ErrorType: classifyXRPError(getErr), + Message: getErr.Error(), + } + } + } + }() + } + + go func() { + defer close(jobs) + for i, num := range blockNumbers { + select { + case <-ctx.Done(): + return + case jobs <- job{index: i, num: num}: + } + } + }() + + wg.Wait() + if ctx.Err() != nil { + return nil, ctx.Err() + } + return results, firstBlockError(results) +} + +func (x *XRPIndexer) IsHealthy() bool { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := x.GetLatestBlockNumber(ctx) + return err == nil +} + +func (x *XRPIndexer) convertLedger(ledger *xrp.Ledger, fallbackNumber uint64) (*types.Block, error) { + if ledger == nil { + return nil, fmt.Errorf("xrp ledger is nil") + } + + number := fallbackNumber + if strings.TrimSpace(ledger.LedgerIndex.String()) != "" { + parsed, err := strconv.ParseUint(ledger.LedgerIndex.String(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid xrp ledger_index %q: %w", ledger.LedgerIndex.String(), err) + } + number = parsed + } + + timestamp := xrpLedgerTime(ledger.CloseTime) + txs := make([]types.Transaction, 0, len(ledger.Transactions)) + blockHash := strings.TrimSpace(ledger.LedgerHash) + for txIndex, tx := range ledger.Transactions { + converted, ok := x.convertTransaction(tx, number, blockHash, txIndex, timestamp) + if !ok { + continue + } + txs = append(txs, converted) + } + + return &types.Block{ + Number: number, + Hash: ledger.LedgerHash, + ParentHash: ledger.ParentHash, + Timestamp: timestamp, + Transactions: txs, + }, nil +} + +func (x *XRPIndexer) convertTransaction( + tx xrp.Transaction, + ledgerIndex uint64, + blockHash string, + txIndex int, + ledgerTimestamp uint64, +) (types.Transaction, bool) { + meta := tx.Meta + if meta == nil { + meta = tx.MetaData + } + if meta == nil || !strings.EqualFold(strings.TrimSpace(meta.TransactionResult), "tesSUCCESS") { + return types.Transaction{}, false + } + + switch strings.ToLower(strings.TrimSpace(tx.TransactionType)) { + case "payment": + return x.convertPaymentTransaction(tx, meta, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + case "accountdelete": + return x.convertAccountDeleteTransaction(tx, meta, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + case "checkcash": + return x.convertCheckCashTransaction(tx, meta, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + case "escrowfinish": + return x.convertEscrowFinishTransaction(tx, meta, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + case "clawback": + return x.convertClawbackTransaction(tx, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + default: + return types.Transaction{}, false + } +} + +func classifyXRPError(err error) ErrorType { + if err == nil { + return ErrorTypeUnknown + } + msg := strings.ToLower(err.Error()) + switch { + case strings.Contains(msg, "not found"): + return ErrorTypeBlockNotFound + case strings.Contains(msg, "timeout"): + return ErrorTypeTimeout + default: + return ErrorTypeUnknown + } +} + +func deliveredAmount(meta *xrp.Meta, fallback any) any { + if meta == nil || meta.DeliveredAmount == nil { + return fallback + } + return meta.DeliveredAmount +} + +func (x *XRPIndexer) convertPaymentTransaction( + tx xrp.Transaction, + meta *xrp.Meta, + ledgerIndex uint64, + blockHash string, + txIndex int, + ledgerTimestamp uint64, +) (types.Transaction, bool) { + from := strings.TrimSpace(tx.Account) + to := strings.TrimSpace(tx.Destination) + if from == "" || to == "" { + return types.Transaction{}, false + } + if !x.isMonitoredTransfer(from, to) { + return types.Transaction{}, false + } + + amount, txType, assetAddress, ok := parseXRPAmount(deliveredAmount(meta, tx.Amount)) + if !ok { + return types.Transaction{}, false + } + + result := x.newBaseTransaction(tx, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + result.FromAddress = normalizeXRPAddress(from) + result.ToAddress = normalizeXRPAddress(to) + result.AssetAddress = assetAddress + result.Amount = amount + result.Type = txType + if sourceAmount, sourceTxType, sourceAssetAddress, ok := x.sourcePaymentDetails(tx, meta); ok { + result.SetMetadata(metadataKeySourceTxType, string(sourceTxType)) + result.SetMetadata(metadataKeySourceAmount, sourceAmount) + if sourceTxType == constant.TxTypeTokenTransfer { + result.SetMetadata(metadataKeySourceAsset, sourceAssetAddress) + } + } + result.DestinationTag = strings.TrimSpace(formatDestinationTag(tx.DestinationTag)) + return result, true +} + +func (x *XRPIndexer) convertAccountDeleteTransaction( + tx xrp.Transaction, + meta *xrp.Meta, + ledgerIndex uint64, + blockHash string, + txIndex int, + ledgerTimestamp uint64, +) (types.Transaction, bool) { + from := strings.TrimSpace(tx.Account) + to := strings.TrimSpace(tx.Destination) + if from == "" || to == "" { + return types.Transaction{}, false + } + if !x.isMonitoredTransfer(from, to) { + return types.Transaction{}, false + } + + amount, ok := accountDeleteAmount(meta, to) + if !ok { + return types.Transaction{}, false + } + + result := x.newBaseTransaction(tx, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + result.FromAddress = normalizeXRPAddress(from) + result.ToAddress = normalizeXRPAddress(to) + result.Amount = amount + result.Type = constant.TxTypeNativeTransfer + setMetadataString(&result, types.MetadataKeySubtype, "account_delete") + result.DestinationTag = strings.TrimSpace(formatDestinationTag(tx.DestinationTag)) + return result, true +} + +func (x *XRPIndexer) convertCheckCashTransaction( + tx xrp.Transaction, + meta *xrp.Meta, + ledgerIndex uint64, + blockHash string, + txIndex int, + ledgerTimestamp uint64, +) (types.Transaction, bool) { + if meta == nil || meta.DeliveredAmount == nil { + return types.Transaction{}, false + } + + checkNode := findLedgerNode(meta, "check", func(node *xrp.LedgerNode) bool { + return strings.EqualFold(strings.TrimSpace(node.LedgerIndex), strings.TrimSpace(tx.CheckID)) + }) + fields := ledgerNodeFields(checkNode) + if fields == nil { + return types.Transaction{}, false + } + + from := strings.TrimSpace(fields.Account) + to := strings.TrimSpace(tx.Account) + if from == "" || to == "" { + return types.Transaction{}, false + } + if !x.isMonitoredTransfer(from, to) { + return types.Transaction{}, false + } + + amount, txType, assetAddress, ok := parseXRPAmount(meta.DeliveredAmount) + if !ok { + return types.Transaction{}, false + } + + result := x.newBaseTransaction(tx, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + result.FromAddress = normalizeXRPAddress(from) + result.ToAddress = normalizeXRPAddress(to) + result.AssetAddress = assetAddress + result.Amount = amount + result.Type = txType + setMetadataString(&result, types.MetadataKeySubtype, "check_cash") + setMetadataString(&result, types.MetadataKeyCheckID, strings.TrimSpace(tx.CheckID)) + result.DestinationTag = strings.TrimSpace(formatDestinationTag(fields.DestinationTag)) + return result, true +} + +func (x *XRPIndexer) convertEscrowFinishTransaction( + tx xrp.Transaction, + meta *xrp.Meta, + ledgerIndex uint64, + blockHash string, + txIndex int, + ledgerTimestamp uint64, +) (types.Transaction, bool) { + owner := normalizeXRPAddress(strings.TrimSpace(tx.Owner)) + offerSequence := uint32String(tx.OfferSequence) + escrowNode := findLedgerNode(meta, "escrow", func(node *xrp.LedgerNode) bool { + fields := ledgerNodeFields(node) + if fields == nil { + return false + } + if owner != "" && !strings.EqualFold(normalizeXRPAddress(fields.Account), owner) { + return false + } + return true + }) + fields := ledgerNodeFields(escrowNode) + if fields == nil { + return types.Transaction{}, false + } + + if owner == "" { + owner = normalizeXRPAddress(fields.Account) + } + to := normalizeXRPAddress(fields.Destination) + if owner == "" || to == "" { + return types.Transaction{}, false + } + if !x.isMonitoredTransfer(owner, to) { + return types.Transaction{}, false + } + + amount, txType, assetAddress, ok := parseXRPAmount(fields.Amount) + if !ok || txType != constant.TxTypeNativeTransfer || assetAddress != "" { + return types.Transaction{}, false + } + + result := x.newBaseTransaction(tx, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + result.FromAddress = owner + result.ToAddress = to + result.Amount = amount + result.Type = constant.TxTypeNativeTransfer + setMetadataString(&result, types.MetadataKeySubtype, "escrow_finish") + setMetadataString(&result, types.MetadataKeyEscrowOwner, owner) + setMetadataString(&result, types.MetadataKeyEscrowSequence, offerSequence) + result.DestinationTag = strings.TrimSpace(formatDestinationTag(fields.DestinationTag)) + return result, true +} + +func (x *XRPIndexer) convertClawbackTransaction( + tx xrp.Transaction, + ledgerIndex uint64, + blockHash string, + txIndex int, + ledgerTimestamp uint64, +) (types.Transaction, bool) { + if strings.TrimSpace(tx.Holder) != "" { + return types.Transaction{}, false + } + + parsed, ok := parseXRPIssuedAmount(tx.Amount) + if !ok { + return types.Transaction{}, false + } + holder := normalizeXRPAddress(parsed.Issuer) + if holder == "" || !x.isMonitoredAddress(holder) { + return types.Transaction{}, false + } + + result := x.newBaseTransaction(tx, ledgerIndex, blockHash, txIndex, ledgerTimestamp) + result.FromAddress = holder + result.ToAddress = xrpBurnAddress + result.AssetAddress = formatXRPIssuedCurrency(tx.Account, parsed.Currency) + result.Amount = strings.TrimSpace(parsed.Value) + result.Type = constant.TxTypeTokenTransfer + setMetadataString(&result, types.MetadataKeySubtype, "clawback") + return result, true +} + +func (x *XRPIndexer) newBaseTransaction( + tx xrp.Transaction, + ledgerIndex uint64, + blockHash string, + txIndex int, + ledgerTimestamp uint64, +) types.Transaction { + timestamp := ledgerTimestamp + if tx.Date != nil && *tx.Date > 0 { + timestamp = xrpLedgerTime(*tx.Date) + } + + return types.Transaction{ + TxHash: strings.TrimSpace(tx.Hash), + NetworkId: x.config.NetworkId, + BlockNumber: ledgerIndex, + BlockHash: blockHash, + TransferIndex: fmt.Sprintf("%d", txIndex), + TxFee: dropsToXRP(strings.TrimSpace(tx.Fee)), + Timestamp: timestamp, + Confirmations: 1, + Status: types.StatusConfirmed, + } +} + +func (x *XRPIndexer) isMonitoredAddress(address string) bool { + if x.pubkeyStore == nil { + return true + } + address = normalizeXRPAddress(address) + return address != "" && x.pubkeyStore.Exist(enum.NetworkTypeXRP, address) +} + +func (x *XRPIndexer) sourcePaymentDetails( + tx xrp.Transaction, + meta *xrp.Meta, +) (string, constant.TxType, string, bool) { + sender := normalizeXRPAddress(tx.Account) + if sender == "" || meta == nil { + return "", "", "", false + } + + if amount, assetAddress, ok := xrpSourceIssuedAmount(meta, sender); ok { + return amount, constant.TxTypeTokenTransfer, assetAddress, true + } + if amount, ok := xrpSourceXRPAmount(meta, sender, tx.Fee); ok { + return amount, constant.TxTypeNativeTransfer, "", true + } + + return "", "", "", false +} + +func xrpSourceXRPAmount(meta *xrp.Meta, account string, fee string) (string, bool) { + account = normalizeXRPAddress(account) + if meta == nil || account == "" { + return "", false + } + + feeValue, ok := parseXRPNumericDecimal(fee) + if !ok { + return "", false + } + + for _, affected := range meta.AffectedNodes { + for _, node := range []*xrp.LedgerNode{affected.ModifiedNode, affected.CreatedNode} { + if node == nil || !strings.EqualFold(strings.TrimSpace(node.LedgerEntryType), "accountroot") { + continue + } + fields := ledgerNodeFields(node) + if fields == nil || !strings.EqualFold(normalizeXRPAddress(fields.Account), account) { + continue + } + + finalBalance, ok := parseXRPNumericDecimal(xrpNumericString(fields.Balance)) + if !ok { + continue + } + + var previousBalance decimal.Decimal + if node.PreviousFields == nil { + continue + } + previousBalance, ok = parseXRPNumericDecimal(xrpNumericString(node.PreviousFields.Balance)) + if !ok { + continue + } + + spentDrops := previousBalance.Sub(finalBalance).Sub(feeValue) + if spentDrops.Cmp(decimal.Zero) <= 0 { + continue + } + + return spentDrops.Div(decimal.NewFromInt(1_000_000)).String(), true + } + } + + return "", false +} + +func xrpSourceIssuedAmount(meta *xrp.Meta, account string) (string, string, bool) { + account = normalizeXRPAddress(account) + if meta == nil || account == "" { + return "", "", false + } + + amountByAsset := make(map[string]decimal.Decimal) + for _, affected := range meta.AffectedNodes { + for _, node := range []*xrp.LedgerNode{affected.DeletedNode, affected.ModifiedNode, affected.CreatedNode} { + if node == nil || !strings.EqualFold(strings.TrimSpace(node.LedgerEntryType), "ripplestate") { + continue + } + + lowAccount, highAccount, currency, ok := xrpRippleStateParticipants(node) + if !ok || (account != lowAccount && account != highAccount) { + continue + } + + previousBalance, finalBalance, balanceCurrency, ok := xrpRippleStateBalances(node) + if !ok { + continue + } + if currency == "" { + currency = balanceCurrency + } + if currency == "" { + continue + } + + var previousHeld, finalHeld decimal.Decimal + var issuer string + switch account { + case lowAccount: + issuer = highAccount + if previousBalance.Cmp(decimal.Zero) > 0 { + previousHeld = previousBalance + } + if finalBalance.Cmp(decimal.Zero) > 0 { + finalHeld = finalBalance + } + case highAccount: + issuer = lowAccount + if previousBalance.Cmp(decimal.Zero) < 0 { + previousHeld = previousBalance.Neg() + } + if finalBalance.Cmp(decimal.Zero) < 0 { + finalHeld = finalBalance.Neg() + } + } + + spent := previousHeld.Sub(finalHeld) + if spent.Cmp(decimal.Zero) <= 0 { + continue + } + + assetAddress := formatXRPIssuedCurrency(issuer, currency) + if assetAddress == "" { + continue + } + amountByAsset[assetAddress] = amountByAsset[assetAddress].Add(spent) + } + } + + var ( + bestAsset string + bestAmount decimal.Decimal + ) + for assetAddress, amount := range amountByAsset { + if amount.Cmp(bestAmount) > 0 { + bestAsset = assetAddress + bestAmount = amount + } + } + if bestAsset == "" || bestAmount.Cmp(decimal.Zero) <= 0 { + return "", "", false + } + + return bestAmount.String(), bestAsset, true +} + +func xrpRippleStateParticipants(node *xrp.LedgerNode) (string, string, string, bool) { + for _, fields := range []*xrp.LedgerFields{node.FinalFields, node.NewFields, node.PreviousFields} { + if fields == nil { + continue + } + + lowAccount, lowCurrency, lowOK := xrpLimitAccount(fields.LowLimit) + highAccount, highCurrency, highOK := xrpLimitAccount(fields.HighLimit) + if !lowOK || !highOK { + continue + } + + currency := lowCurrency + if currency == "" { + currency = highCurrency + } + return lowAccount, highAccount, currency, true + } + + return "", "", "", false +} + +func xrpRippleStateBalances(node *xrp.LedgerNode) (decimal.Decimal, decimal.Decimal, string, bool) { + if node == nil { + return decimal.Zero, decimal.Zero, "", false + } + + var ( + previousBalance decimal.Decimal + finalBalance decimal.Decimal + currency string + ok bool + ) + + switch { + case node.FinalFields != nil: + currency, finalBalance, ok = xrpIssuedBalance(node.FinalFields.Balance) + if !ok { + return decimal.Zero, decimal.Zero, "", false + } + previousBalance = finalBalance + if node.PreviousFields != nil && node.PreviousFields.Balance != nil { + if previousCurrency, previous, previousOK := xrpIssuedBalance(node.PreviousFields.Balance); previousOK { + previousBalance = previous + if currency == "" { + currency = previousCurrency + } + } + } + case node.NewFields != nil: + currency, finalBalance, ok = xrpIssuedBalance(node.NewFields.Balance) + if !ok { + return decimal.Zero, decimal.Zero, "", false + } + case node.PreviousFields != nil: + currency, previousBalance, ok = xrpIssuedBalance(node.PreviousFields.Balance) + if !ok { + return decimal.Zero, decimal.Zero, "", false + } + default: + return decimal.Zero, decimal.Zero, "", false + } + + return previousBalance, finalBalance, currency, true +} + +func xrpLimitAccount(raw any) (string, string, bool) { + parsed, ok := xrpIssuedAmountValue(raw) + if !ok { + return "", "", false + } + + account := normalizeXRPAddress(parsed.Issuer) + currency := strings.TrimSpace(parsed.Currency) + if account == "" || currency == "" { + return "", "", false + } + + return account, currency, true +} + +func xrpIssuedBalance(raw any) (string, decimal.Decimal, bool) { + parsed, ok := xrpIssuedAmountValue(raw) + if !ok { + return "", decimal.Zero, false + } + + value, ok := parseXRPNumericDecimal(parsed.Value) + if !ok { + return "", decimal.Zero, false + } + + currency := strings.TrimSpace(parsed.Currency) + if currency == "" { + return "", decimal.Zero, false + } + + return currency, value, true +} + +func xrpIssuedAmountValue(raw any) (xrp.IssuedCurrencyAmount, bool) { + switch value := raw.(type) { + case map[string]any: + var parsed xrp.IssuedCurrencyAmount + data, err := json.Marshal(value) + if err != nil { + return xrp.IssuedCurrencyAmount{}, false + } + if err := json.Unmarshal(data, &parsed); err != nil { + return xrp.IssuedCurrencyAmount{}, false + } + if strings.TrimSpace(parsed.Currency) == "" || strings.TrimSpace(parsed.Value) == "" { + return xrp.IssuedCurrencyAmount{}, false + } + return parsed, true + case xrp.IssuedCurrencyAmount: + if strings.TrimSpace(value.Currency) == "" || strings.TrimSpace(value.Value) == "" { + return xrp.IssuedCurrencyAmount{}, false + } + return value, true + default: + return xrp.IssuedCurrencyAmount{}, false + } +} + +func parseXRPNumericDecimal(raw string) (decimal.Decimal, bool) { + raw = strings.TrimSpace(raw) + if raw == "" { + return decimal.Zero, false + } + + value, err := decimal.NewFromString(raw) + if err != nil { + return decimal.Zero, false + } + return value, true +} + +func accountDeleteAmount(meta *xrp.Meta, destination string) (string, bool) { + destination = normalizeXRPAddress(destination) + if meta == nil || destination == "" { + return "", false + } + for _, affected := range meta.AffectedNodes { + for _, node := range []*xrp.LedgerNode{affected.ModifiedNode, affected.CreatedNode} { + if node == nil || !strings.EqualFold(strings.TrimSpace(node.LedgerEntryType), "accountroot") { + continue + } + fields := ledgerNodeFields(node) + if fields == nil || !strings.EqualFold(normalizeXRPAddress(fields.Account), destination) { + continue + } + finalBalance := xrpNumericString(fields.Balance) + previousBalance := "" + if node.PreviousFields != nil { + previousBalance = xrpNumericString(node.PreviousFields.Balance) + } + if finalBalance == "" || previousBalance == "" { + continue + } + delta := decimal.RequireFromString(finalBalance).Sub(decimal.RequireFromString(previousBalance)) + if delta.Cmp(decimal.Zero) <= 0 { + continue + } + return delta.Div(decimal.NewFromInt(1_000_000)).String(), true + } + } + return "", false +} + +func findLedgerNode(meta *xrp.Meta, entryType string, match func(node *xrp.LedgerNode) bool) *xrp.LedgerNode { + if meta == nil { + return nil + } + for _, affected := range meta.AffectedNodes { + for _, node := range []*xrp.LedgerNode{affected.DeletedNode, affected.ModifiedNode, affected.CreatedNode} { + if node == nil || !strings.EqualFold(strings.TrimSpace(node.LedgerEntryType), entryType) { + continue + } + if match == nil || match(node) { + return node + } + } + } + return nil +} + +func ledgerNodeFields(node *xrp.LedgerNode) *xrp.LedgerFields { + if node == nil { + return nil + } + switch { + case node.FinalFields != nil: + return node.FinalFields + case node.NewFields != nil: + return node.NewFields + default: + return node.PreviousFields + } +} + +func parseXRPAmount(raw any) (amount string, txType constant.TxType, assetAddress string, ok bool) { + switch value := raw.(type) { + case string: + return dropsToXRP(value).String(), constant.TxTypeNativeTransfer, "", true + case map[string]any: + var parsed xrp.IssuedCurrencyAmount + data, err := json.Marshal(value) + if err != nil { + return "", "", "", false + } + if err := json.Unmarshal(data, &parsed); err != nil { + return "", "", "", false + } + if strings.TrimSpace(parsed.Issuer) == "" || strings.TrimSpace(parsed.Currency) == "" || strings.TrimSpace(parsed.Value) == "" { + return "", "", "", false + } + return strings.TrimSpace(parsed.Value), constant.TxTypeTokenTransfer, formatXRPIssuedCurrency(parsed.Issuer, parsed.Currency), true + case xrp.IssuedCurrencyAmount: + if strings.TrimSpace(value.Issuer) == "" || strings.TrimSpace(value.Currency) == "" || strings.TrimSpace(value.Value) == "" { + return "", "", "", false + } + return strings.TrimSpace(value.Value), constant.TxTypeTokenTransfer, formatXRPIssuedCurrency(value.Issuer, value.Currency), true + default: + return "", "", "", false + } +} + +func xrpNumericString(raw any) string { + switch value := raw.(type) { + case string: + return strings.TrimSpace(value) + case json.Number: + return strings.TrimSpace(value.String()) + default: + return "" + } +} + +func parseXRPIssuedAmount(raw any) (xrp.IssuedCurrencyAmount, bool) { + switch value := raw.(type) { + case map[string]any: + var parsed xrp.IssuedCurrencyAmount + data, err := json.Marshal(value) + if err != nil { + return xrp.IssuedCurrencyAmount{}, false + } + if err := json.Unmarshal(data, &parsed); err != nil { + return xrp.IssuedCurrencyAmount{}, false + } + if strings.TrimSpace(parsed.Issuer) == "" || strings.TrimSpace(parsed.Currency) == "" || strings.TrimSpace(parsed.Value) == "" { + return xrp.IssuedCurrencyAmount{}, false + } + return parsed, true + case xrp.IssuedCurrencyAmount: + if strings.TrimSpace(value.Issuer) == "" || strings.TrimSpace(value.Currency) == "" || strings.TrimSpace(value.Value) == "" { + return xrp.IssuedCurrencyAmount{}, false + } + return value, true + default: + return xrp.IssuedCurrencyAmount{}, false + } +} + +func formatXRPIssuedCurrency(issuer string, currency string) string { + return normalizeXRPAddress(issuer) + ":" + strings.ToUpper(strings.TrimSpace(currency)) +} + +func dropsToXRP(drops string) decimal.Decimal { + if strings.TrimSpace(drops) == "" { + return decimal.Zero + } + return decimal.RequireFromString(strings.TrimSpace(drops)).Div(decimal.NewFromInt(1_000_000)) +} + +func xrpLedgerTime(v int64) uint64 { + if v <= 0 { + return 0 + } + return uint64(v + xrpEpochOffset) +} + +func normalizeXRPAddress(address string) string { + return strings.TrimSpace(address) +} + +func setMetadataString(tx *types.Transaction, key string, value string) { + if strings.TrimSpace(value) == "" { + return + } + tx.SetMetadata(key, strings.TrimSpace(value)) +} + +func formatDestinationTag(tag *uint32) string { + if tag == nil { + return "" + } + return strconv.FormatUint(uint64(*tag), 10) +} + +func uint32String(v *uint32) string { + if v == nil { + return "" + } + return strconv.FormatUint(uint64(*v), 10) +} + +func firstBlockError(results []BlockResult) error { + for _, result := range results { + if result.Error != nil { + return fmt.Errorf("block %d: %s", result.Number, result.Error.Message) + } + } + return nil +} diff --git a/internal/indexer/xrp_test.go b/internal/indexer/xrp_test.go new file mode 100644 index 0000000..6bd07c6 --- /dev/null +++ b/internal/indexer/xrp_test.go @@ -0,0 +1,730 @@ +package indexer + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/fystack/multichain-indexer/internal/rpc" + "github.com/fystack/multichain-indexer/internal/rpc/xrp" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/enum" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type mockXRPPubkeyStore struct { + addresses map[string]struct{} +} + +func (m mockXRPPubkeyStore) Exist(_ enum.NetworkType, address string) bool { + _, ok := m.addresses[address] + return ok +} + +type mockXRPAPI struct { + ledgers map[uint64]*xrp.Ledger +} + +var _ xrp.XRPLAPI = (*mockXRPAPI)(nil) + +func (m *mockXRPAPI) CallRPC(context.Context, string, any) (*rpc.RPCResponse, error) { return nil, nil } +func (m *mockXRPAPI) Do(context.Context, string, string, any, map[string]string) ([]byte, error) { + return nil, nil +} +func (m *mockXRPAPI) GetNetworkType() string { return rpc.NetworkXRP } +func (m *mockXRPAPI) GetClientType() string { return rpc.ClientTypeRPC } +func (m *mockXRPAPI) GetURL() string { return "http://xrpl.test" } +func (m *mockXRPAPI) Close() error { return nil } + +func (m *mockXRPAPI) GetLatestLedgerIndex(context.Context) (uint64, error) { return 100, nil } +func (m *mockXRPAPI) GetLedgerByIndex(_ context.Context, ledgerIndex uint64) (*xrp.Ledger, error) { + return m.ledgers[ledgerIndex], nil +} +func (m *mockXRPAPI) BatchGetLedgersByIndex(_ context.Context, ledgerIndexes []uint64) (map[uint64]*xrp.Ledger, error) { + out := make(map[uint64]*xrp.Ledger, len(ledgerIndexes)) + for _, ledgerIndex := range ledgerIndexes { + out[ledgerIndex] = m.ledgers[ledgerIndex] + } + return out, nil +} + +const xrpMainnetRPC = "https://xrplcluster.com" + +func newTestXRPLiveIndexer(t *testing.T) (*XRPIndexer, *xrp.Client) { + t.Helper() + + client := xrp.NewClient(xrpMainnetRPC, nil, 30*time.Second, nil) + t.Cleanup(func() { + _ = client.Close() + }) + + failover := rpc.NewFailover[xrp.XRPLAPI](nil) + require.NoError(t, failover.AddProvider(&rpc.Provider{ + Name: "xrp-mainnet", + URL: xrpMainnetRPC, + Network: "xrp_mainnet", + ClientType: rpc.ClientTypeRPC, + Client: client, + State: rpc.StateHealthy, + })) + + return NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{NetworkId: "xrp_mainnet"}, + failover, + nil, + ), client +} + +func findXRPBlockTransaction(t *testing.T, block *types.Block, txHash string) types.Transaction { + t.Helper() + + for _, tx := range block.Transactions { + if tx.TxHash == txHash { + return tx + } + } + + t.Fatalf("transaction %s not found in block %d", txHash, block.Number) + return types.Transaction{} +} + +func findXRPLedgerTransactionIndex(t *testing.T, ledger *xrp.Ledger, txHash string) string { + t.Helper() + + for i, tx := range ledger.Transactions { + if tx.Hash == txHash { + return fmt.Sprintf("%d", i) + } + } + + t.Fatalf("transaction %s not found in ledger %s", txHash, ledger.LedgerIndex.String()) + return "" +} + +func TestXRPConvertLedger_ParsesNativeAndIssuedPayments(t *testing.T) { + t.Parallel() + + destinationTag := uint32(778899) + idx := NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{ + NetworkId: "xrp_mainnet", + }, + nil, + mockXRPPubkeyStore{addresses: map[string]struct{}{ + "rDest": {}, + }}, + ) + + ledger := &xrp.Ledger{ + LedgerHash: "LEDGER_HASH", + ParentHash: "PARENT_HASH", + LedgerIndex: "100", + CloseTime: 100, + Transactions: []xrp.Transaction{ + { + Hash: "TX_NATIVE", + Account: "rSource", + Destination: "rDest", + DestinationTag: &destinationTag, + TransactionType: "Payment", + Fee: "12", + Amount: "2500000", + Meta: &xrp.Meta{ + TransactionResult: "tesSUCCESS", + }, + }, + { + Hash: "TX_TOKEN", + Account: "rIssuerSource", + Destination: "rDest", + TransactionType: "Payment", + Fee: "20", + Amount: map[string]any{ + "currency": "USD", + "issuer": "rIssuer", + "value": "5.25", + }, + Meta: &xrp.Meta{ + TransactionResult: "tesSUCCESS", + DeliveredAmount: map[string]any{ + "currency": "USD", + "issuer": "rIssuer", + "value": "5.20", + }, + }, + }, + { + Hash: "TX_SKIP", + Account: "rSource", + Destination: "rDest", + TransactionType: "OfferCreate", + Fee: "10", + Meta: &xrp.Meta{ + TransactionResult: "tesSUCCESS", + }, + }, + }, + } + + block, err := idx.convertLedger(ledger, 100) + require.NoError(t, err) + require.Len(t, block.Transactions, 2) + + nativeTx := block.Transactions[0] + assert.Equal(t, "TX_NATIVE", nativeTx.TxHash) + assert.Equal(t, "2.5", nativeTx.Amount) + assert.Equal(t, constant.TxTypeNativeTransfer, nativeTx.Type) + assert.Equal(t, "", nativeTx.AssetAddress) + assert.Equal(t, "LEDGER_HASH", nativeTx.BlockHash) + assert.Equal(t, "0", nativeTx.TransferIndex) + destinationTagValue := nativeTx.DestinationTag + assert.Equal(t, "778899", destinationTagValue) + assert.Equal(t, "0.000012", nativeTx.TxFee.String()) + + tokenTx := block.Transactions[1] + assert.Equal(t, "TX_TOKEN", tokenTx.TxHash) + assert.Equal(t, constant.TxTypeTokenTransfer, tokenTx.Type) + assert.Equal(t, "5.20", tokenTx.Amount) + assert.Equal(t, "rIssuer:USD", tokenTx.AssetAddress) + assert.Equal(t, "LEDGER_HASH", tokenTx.BlockHash) + assert.Equal(t, "1", tokenTx.TransferIndex) +} + +func TestXRPConvertPaymentTransaction_StoresSourceSideRoutingForPathPayment(t *testing.T) { + t.Parallel() + + idx := NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{NetworkId: "xrp_mainnet"}, + nil, + mockXRPPubkeyStore{addresses: map[string]struct{}{"rDest": {}}}, + ) + + tx, ok := idx.convertPaymentTransaction( + xrp.Transaction{ + Hash: "TX_PATH", + Account: "rSender", + Destination: "rDest", + TransactionType: "Payment", + Fee: "12", + Amount: "5000000", + }, + &xrp.Meta{ + TransactionResult: "tesSUCCESS", + DeliveredAmount: "5000000", + AffectedNodes: []xrp.Node{ + { + ModifiedNode: &xrp.LedgerNode{ + LedgerEntryType: "RippleState", + FinalFields: &xrp.LedgerFields{ + Balance: map[string]any{ + "currency": "USD", + "value": "90", + }, + LowLimit: map[string]any{ + "currency": "USD", + "issuer": "rSender", + "value": "1000000000", + }, + HighLimit: map[string]any{ + "currency": "USD", + "issuer": "rIssuer", + "value": "0", + }, + }, + PreviousFields: &xrp.LedgerFields{ + Balance: map[string]any{ + "currency": "USD", + "value": "100", + }, + }, + }, + }, + }, + }, + 105, + "LEDGER_HASH", + 0, + 123, + ) + require.True(t, ok) + assert.Equal(t, "5", tx.Amount) + assert.Equal(t, constant.TxTypeNativeTransfer, tx.Type) + assert.Equal(t, "", tx.AssetAddress) + assert.Equal(t, string(constant.TxTypeTokenTransfer), tx.GetMetadataString(metadataKeySourceTxType)) + assert.Equal(t, "10", tx.GetMetadataString(metadataKeySourceAmount)) + assert.Equal(t, "rIssuer:USD", tx.GetMetadataString(metadataKeySourceAsset)) + + outTx := idx.NormalizeForDirection(tx, types.DirectionOut) + assert.Equal(t, constant.TxTypeTokenTransfer, outTx.Type) + assert.Equal(t, "10", outTx.Amount) + assert.Equal(t, "rIssuer:USD", outTx.AssetAddress) + assert.Equal(t, "", outTx.GetMetadataString(metadataKeySourceTxType)) + assert.Equal(t, "", outTx.GetMetadataString(metadataKeySourceAmount)) + assert.Equal(t, "", outTx.GetMetadataString(metadataKeySourceAsset)) +} + +func TestXRPConvertLedger_RespectsTwoWayIndexing(t *testing.T) { + t.Parallel() + + ledger := &xrp.Ledger{ + LedgerHash: "LEDGER_HASH", + ParentHash: "PARENT_HASH", + LedgerIndex: "5", + CloseTime: 100, + Transactions: []xrp.Transaction{ + { + Hash: "TX_OUT", + Account: "rMonitored", + Destination: "rOther", + TransactionType: "Payment", + Fee: "12", + Amount: "1000000", + Meta: &xrp.Meta{ + TransactionResult: "tesSUCCESS", + }, + }, + }, + } + + idxWithoutTwoWay := NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{NetworkId: "xrp_mainnet"}, + nil, + mockXRPPubkeyStore{addresses: map[string]struct{}{"rMonitored": {}}}, + ) + block, err := idxWithoutTwoWay.convertLedger(ledger, 5) + require.NoError(t, err) + require.Len(t, block.Transactions, 0) + + idxWithTwoWay := NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{ + NetworkId: "xrp_mainnet", + TwoWayIndexing: true, + }, + nil, + mockXRPPubkeyStore{addresses: map[string]struct{}{"rMonitored": {}}}, + ) + block, err = idxWithTwoWay.convertLedger(ledger, 5) + require.NoError(t, err) + require.Len(t, block.Transactions, 1) +} + +func TestXRPConvertLedger_ParsesAccountDelete(t *testing.T) { + t.Parallel() + + destinationTag := uint32(42) + idx := NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{NetworkId: "xrp_mainnet"}, + nil, + mockXRPPubkeyStore{addresses: map[string]struct{}{"rDest": {}}}, + ) + + block, err := idx.convertLedger(&xrp.Ledger{ + LedgerHash: "LEDGER_HASH", + ParentHash: "PARENT_HASH", + LedgerIndex: "101", + CloseTime: 100, + Transactions: []xrp.Transaction{ + { + Hash: "TX_DELETE", + Account: "rSource", + Destination: "rDest", + DestinationTag: &destinationTag, + TransactionType: "AccountDelete", + Fee: "5000000", + Meta: &xrp.Meta{ + TransactionResult: "tesSUCCESS", + AffectedNodes: []xrp.Node{ + { + ModifiedNode: &xrp.LedgerNode{ + LedgerEntryType: "AccountRoot", + FinalFields: &xrp.LedgerFields{ + Account: "rDest", + Balance: "15000000", + }, + PreviousFields: &xrp.LedgerFields{ + Balance: "10000000", + }, + }, + }, + }, + }, + }, + }, + }, 101) + require.NoError(t, err) + require.Len(t, block.Transactions, 1) + + tx := block.Transactions[0] + assert.Equal(t, "rSource", tx.FromAddress) + assert.Equal(t, "rDest", tx.ToAddress) + assert.Equal(t, "5", tx.Amount) + assert.Equal(t, constant.TxTypeNativeTransfer, tx.Type) + subtype := tx.GetMetadataString(types.MetadataKeySubtype) + assert.Equal(t, "account_delete", subtype) + tag := tx.DestinationTag + assert.Equal(t, "42", tag) +} + +func TestXRPConvertLedger_ParsesCheckCashIssuedAsset(t *testing.T) { + t.Parallel() + + destinationTag := uint32(778899) + idx := NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{NetworkId: "xrp_mainnet"}, + nil, + mockXRPPubkeyStore{addresses: map[string]struct{}{"rCashier": {}}}, + ) + + block, err := idx.convertLedger(&xrp.Ledger{ + LedgerHash: "LEDGER_HASH", + ParentHash: "PARENT_HASH", + LedgerIndex: "102", + CloseTime: 100, + Transactions: []xrp.Transaction{ + { + Hash: "TX_CHECK", + Account: "rCashier", + CheckID: "CHECK1", + TransactionType: "CheckCash", + Fee: "12", + Meta: &xrp.Meta{ + TransactionResult: "tesSUCCESS", + DeliveredAmount: map[string]any{ + "currency": "USD", + "issuer": "rIssuer", + "value": "5.20", + }, + AffectedNodes: []xrp.Node{ + { + DeletedNode: &xrp.LedgerNode{ + LedgerEntryType: "Check", + LedgerIndex: "CHECK1", + FinalFields: &xrp.LedgerFields{ + Account: "rSource", + Destination: "rCashier", + DestinationTag: &destinationTag, + }, + }, + }, + }, + }, + }, + }, + }, 102) + require.NoError(t, err) + require.Len(t, block.Transactions, 1) + + tx := block.Transactions[0] + assert.Equal(t, "rSource", tx.FromAddress) + assert.Equal(t, "rCashier", tx.ToAddress) + assert.Equal(t, "5.20", tx.Amount) + assert.Equal(t, "rIssuer:USD", tx.AssetAddress) + assert.Equal(t, constant.TxTypeTokenTransfer, tx.Type) + subtype := tx.GetMetadataString(types.MetadataKeySubtype) + assert.Equal(t, "check_cash", subtype) + checkID := tx.GetMetadataString(types.MetadataKeyCheckID) + assert.Equal(t, "CHECK1", checkID) + tag := tx.DestinationTag + assert.Equal(t, "778899", tag) +} + +func TestXRPConvertLedger_ParsesEscrowFinish(t *testing.T) { + t.Parallel() + + destinationTag := uint32(12) + offerSequence := uint32(7) + idx := NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{NetworkId: "xrp_mainnet"}, + nil, + mockXRPPubkeyStore{addresses: map[string]struct{}{"rDest": {}}}, + ) + + block, err := idx.convertLedger(&xrp.Ledger{ + LedgerHash: "LEDGER_HASH", + ParentHash: "PARENT_HASH", + LedgerIndex: "103", + CloseTime: 100, + Transactions: []xrp.Transaction{ + { + Hash: "TX_ESCROW", + Account: "rExecutor", + Owner: "rOwner", + OfferSequence: &offerSequence, + TransactionType: "EscrowFinish", + Fee: "12", + Meta: &xrp.Meta{ + TransactionResult: "tesSUCCESS", + AffectedNodes: []xrp.Node{ + { + DeletedNode: &xrp.LedgerNode{ + LedgerEntryType: "Escrow", + FinalFields: &xrp.LedgerFields{ + Account: "rOwner", + Destination: "rDest", + DestinationTag: &destinationTag, + Amount: "2500000", + }, + }, + }, + }, + }, + }, + }, + }, 103) + require.NoError(t, err) + require.Len(t, block.Transactions, 1) + + tx := block.Transactions[0] + assert.Equal(t, "rOwner", tx.FromAddress) + assert.Equal(t, "rDest", tx.ToAddress) + assert.Equal(t, "2.5", tx.Amount) + assert.Equal(t, constant.TxTypeNativeTransfer, tx.Type) + subtype := tx.GetMetadataString(types.MetadataKeySubtype) + assert.Equal(t, "escrow_finish", subtype) + escrowOwner := tx.GetMetadataString(types.MetadataKeyEscrowOwner) + assert.Equal(t, "rOwner", escrowOwner) + escrowSequence := tx.GetMetadataString(types.MetadataKeyEscrowSequence) + assert.Equal(t, "7", escrowSequence) + tag := tx.DestinationTag + assert.Equal(t, "12", tag) +} + +func TestXRPConvertLedger_ParsesClawback(t *testing.T) { + t.Parallel() + + idx := NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{NetworkId: "xrp_mainnet"}, + nil, + mockXRPPubkeyStore{addresses: map[string]struct{}{"rHolder": {}}}, + ) + + block, err := idx.convertLedger(&xrp.Ledger{ + LedgerHash: "LEDGER_HASH", + ParentHash: "PARENT_HASH", + LedgerIndex: "104", + CloseTime: 100, + Transactions: []xrp.Transaction{ + { + Hash: "TX_CLAWBACK", + Account: "rIssuer", + TransactionType: "Clawback", + Fee: "12", + Amount: map[string]any{ + "currency": "USD", + "issuer": "rHolder", + "value": "5.20", + }, + Meta: &xrp.Meta{ + TransactionResult: "tesSUCCESS", + }, + }, + }, + }, 104) + require.NoError(t, err) + require.Len(t, block.Transactions, 1) + + tx := block.Transactions[0] + assert.Equal(t, "rHolder", tx.FromAddress) + assert.Equal(t, xrpBurnAddress, tx.ToAddress) + assert.Equal(t, "rIssuer:USD", tx.AssetAddress) + assert.Equal(t, "5.20", tx.Amount) + assert.Equal(t, constant.TxTypeTokenTransfer, tx.Type) + subtype := tx.GetMetadataString(types.MetadataKeySubtype) + assert.Equal(t, "clawback", subtype) +} + +func TestXRPGetBlocksByNumbers_PreservesInputOrder(t *testing.T) { + t.Parallel() + + api := &mockXRPAPI{ + ledgers: map[uint64]*xrp.Ledger{ + 7: {LedgerHash: "L7", ParentHash: "L6", LedgerIndex: "7", CloseTime: 10}, + 3: {LedgerHash: "L3", ParentHash: "L2", LedgerIndex: "3", CloseTime: 10}, + }, + } + failover := rpc.NewFailover[xrp.XRPLAPI](nil) + require.NoError(t, failover.AddProvider(&rpc.Provider{ + Name: "xrp-test", + URL: "http://xrpl.test", + Network: "xrp_mainnet", + ClientType: rpc.ClientTypeRPC, + Client: api, + State: rpc.StateHealthy, + })) + + idx := NewXRPIndexer( + "xrp_mainnet", + config.ChainConfig{NetworkId: "xrp_mainnet", Throttle: config.Throttle{Concurrency: 2}}, + failover, + nil, + ) + + results, err := idx.GetBlocksByNumbers(context.Background(), []uint64{7, 3}) + require.NoError(t, err) + require.Len(t, results, 2) + assert.Equal(t, uint64(7), results[0].Number) + assert.Equal(t, "L7", results[0].Block.Hash) + assert.Equal(t, uint64(3), results[1].Number) + assert.Equal(t, "L3", results[1].Block.Hash) +} + +func TestXRPMainnetFetchAndParseTransactions(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + type xrpRealTxCase struct { + name string + ledgerIndex uint64 + txHash string + wantType constant.TxType + wantFrom string + wantTo string + wantAmount string + wantAsset string + wantDestinationTag string + wantMetadata map[string]string + } + + testCases := []xrpRealTxCase{ + { + name: "native payment with destination tag", + ledgerIndex: 103233614, + txHash: "F13A325DF5544E7E6F17108B18D0E8198EC161EE26E66345E09E346F154BDD17", + wantType: constant.TxTypeNativeTransfer, + wantFrom: "rMgUZD9LBBrsSLXmAe2naCAWGtTg5KmsT1", + wantTo: "rvEBMEWUUcKNMvXhxzo6Db1pFdrgfi34B", + wantAmount: "8.999987", + wantDestinationTag: "1862722728", + }, + { + name: "issued payment", + ledgerIndex: 94229458, + txHash: "DF31D6B64DFB75600591235BA000B2D2C00C6989A84AB1DD596B5CB9FA8D109B", + wantType: constant.TxTypeTokenTransfer, + wantFrom: "rXPMxDRxMM6JLk8AMVh569iap3TtnjaF3", + wantTo: "rG3cUwFd6UrZocrEDm17b8YZeaHGaNmJBg", + wantAmount: "815", + wantAsset: "rG4kgkvbAiy69t6keLzSBGTgRk8hgJM7Go:4245415652000000000000000000000000000000", + }, + { + name: "account delete", + ledgerIndex: 103056689, + txHash: "BDB1F216067FDC16CA70E585900C72CE75450B1D2524269137C2C4CE59CFA5D5", + wantType: constant.TxTypeNativeTransfer, + wantFrom: "rauksASXjkCF13cpjDGA1DCgWWRm4arJu4", + wantTo: "rMkJJ4HHHBRdeHvE2UXx1xPDh4hnuRW9TZ", + wantAmount: "0.899292", + wantDestinationTag: "1717742333", + wantMetadata: map[string]string{ + types.MetadataKeySubtype: "account_delete", + }, + }, + { + name: "check cash", + ledgerIndex: 103056610, + txHash: "9317A1D9C13A618B8BE74A22D846B6AD70D62364900E8DAA720D79221CFCDF5B", + wantType: constant.TxTypeNativeTransfer, + wantFrom: "rNsvWWTquJ2rWhGcTEx1AEoWte7ZD9VQPe", + wantTo: "ra61uRmrMG1hpVQKhkL6pMwbJqdYcK1FT7", + wantAmount: "4.004482", + wantMetadata: map[string]string{ + types.MetadataKeySubtype: "check_cash", + types.MetadataKeyCheckID: "3C00D77DE88D4C6AA8E3844AA00D68347925D2CB4ABC987E65AAB9D53CDEB4F0", + }, + }, + { + name: "escrow finish", + ledgerIndex: 100398890, + txHash: "2861C22727E48831F8D45E304F15631C8927D52E46E6ED5060929FE2C7A83A34", + wantType: constant.TxTypeNativeTransfer, + wantFrom: "rw37cgPnjt57zqKgFuwqZExsiq79A3kZuP", + wantTo: "raftCiiYJoU5UrkBKjAoHfLuo6iZDotLr6", + wantAmount: "58.974", + wantMetadata: map[string]string{ + types.MetadataKeySubtype: "escrow_finish", + types.MetadataKeyEscrowOwner: "rw37cgPnjt57zqKgFuwqZExsiq79A3kZuP", + types.MetadataKeyEscrowSequence: "76046814", + }, + }, + { + name: "clawback", + ledgerIndex: 102983492, + txHash: "ADCF519C04EB8AC273EE1C136258D8A62833E2E57AE3A56EB9C6595EB272FB05", + wantType: constant.TxTypeTokenTransfer, + wantFrom: "rLqBaPeR8xe4d1RJ7tBg6iFVyJjpo1rS2T", + wantTo: xrpBurnAddress, + wantAmount: "200", + wantAsset: "r4WspQvEcvCjLQz6cDAxTS5vV6rQ9GuKsF:CLW", + wantMetadata: map[string]string{ + types.MetadataKeySubtype: "clawback", + }, + }, + } + + enabled := false + for _, tc := range testCases { + if tc.ledgerIndex != 0 && tc.txHash != "" { + enabled = true + break + } + } + if !enabled { + t.Skip("hardcoded real mainnet test cases are empty") + } + + idx, client := newTestXRPLiveIndexer(t) + + for _, tc := range testCases { + tc := tc + if tc.ledgerIndex == 0 || tc.txHash == "" { + continue + } + + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + ledger, err := client.GetLedgerByIndex(ctx, tc.ledgerIndex) + require.NoError(t, err, "should fetch real mainnet ledger %d", tc.ledgerIndex) + require.NotNil(t, ledger) + + block, err := idx.GetBlock(ctx, tc.ledgerIndex) + require.NoError(t, err, "should parse real mainnet ledger %d", tc.ledgerIndex) + require.NotNil(t, block) + + tx := findXRPBlockTransaction(t, block, tc.txHash) + require.NotEmpty(t, tx.TxHash) + require.NotEmpty(t, tx.FromAddress) + require.NotEmpty(t, tx.ToAddress) + require.NotEmpty(t, tx.Type) + + assert.Equal(t, tc.txHash, tx.TxHash) + assert.Equal(t, tc.wantFrom, tx.FromAddress) + assert.Equal(t, tc.wantTo, tx.ToAddress) + assert.Equal(t, tc.wantAmount, tx.Amount) + assert.Equal(t, tc.wantAsset, tx.AssetAddress) + assert.Equal(t, tc.wantType, tx.Type) + assert.Equal(t, block.Hash, tx.BlockHash) + assert.Equal(t, findXRPLedgerTransactionIndex(t, ledger, tc.txHash), tx.TransferIndex) + assert.Equal(t, tc.wantDestinationTag, tx.DestinationTag) + + for key, want := range tc.wantMetadata { + assert.Equal(t, want, tx.GetMetadataString(key), "metadata %s", key) + } + }) + } +} diff --git a/internal/rpc/stellar/api.go b/internal/rpc/stellar/api.go new file mode 100644 index 0000000..a2eb940 --- /dev/null +++ b/internal/rpc/stellar/api.go @@ -0,0 +1,18 @@ +package stellar + +import ( + "context" + + "github.com/fystack/multichain-indexer/internal/rpc" +) + +type StellarAPI interface { + rpc.NetworkClient + + GetLatestLedgerSequence(ctx context.Context) (uint64, error) + GetLedger(ctx context.Context, sequence uint64) (*Ledger, error) + GetPaymentsByLedger(ctx context.Context, sequence uint64, cursor string, limit int) (*PaymentsPage, error) + GetOperationsByLedger(ctx context.Context, sequence uint64, cursor string, limit int) (*OperationsPage, error) + GetEffectsByOperation(ctx context.Context, operationID string, cursor string, limit int) (*EffectsPage, error) + GetTransaction(ctx context.Context, hash string) (*Transaction, error) +} diff --git a/internal/rpc/stellar/client.go b/internal/rpc/stellar/client.go new file mode 100644 index 0000000..9f6d879 --- /dev/null +++ b/internal/rpc/stellar/client.go @@ -0,0 +1,173 @@ +package stellar + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/fystack/multichain-indexer/internal/rpc" + "github.com/fystack/multichain-indexer/pkg/ratelimiter" +) + +type Client struct { + base *rpc.BaseClient +} + +func NewClient( + baseURL string, + auth *rpc.AuthConfig, + timeout time.Duration, + rl *ratelimiter.PooledRateLimiter, +) *Client { + return &Client{ + base: rpc.NewBaseClient(baseURL, rpc.NetworkStellar, rpc.ClientTypeREST, auth, timeout, rl), + } +} + +func (c *Client) CallRPC(ctx context.Context, method string, params any) (*rpc.RPCResponse, error) { + return c.base.CallRPC(ctx, method, params) +} + +func (c *Client) Do( + ctx context.Context, + method, endpoint string, + body any, + params map[string]string, +) ([]byte, error) { + return c.base.Do(ctx, method, endpoint, body, params) +} + +func (c *Client) GetNetworkType() string { return c.base.GetNetworkType() } +func (c *Client) GetClientType() string { return c.base.GetClientType() } +func (c *Client) GetURL() string { return c.base.GetURL() } +func (c *Client) Close() error { return c.base.Close() } +func (c *Client) SetCustomHeaders(headers map[string]string) { + c.base.SetCustomHeaders(headers) +} + +func (c *Client) GetLatestLedgerSequence(ctx context.Context) (uint64, error) { + raw, err := c.base.Do(ctx, http.MethodGet, "", nil, nil) + if err != nil { + return 0, fmt.Errorf("get stellar root failed: %w", err) + } + + var root Root + if err := json.Unmarshal(raw, &root); err != nil { + return 0, fmt.Errorf("decode stellar root failed: %w", err) + } + if root.HistoryLatestLedger == 0 { + return 0, fmt.Errorf("stellar root returned empty history_latest_ledger") + } + return root.HistoryLatestLedger, nil +} + +func (c *Client) GetLedger(ctx context.Context, sequence uint64) (*Ledger, error) { + raw, err := c.base.Do(ctx, http.MethodGet, fmt.Sprintf("/ledgers/%d", sequence), nil, nil) + if err != nil { + return nil, fmt.Errorf("get stellar ledger %d failed: %w", sequence, err) + } + + var ledger Ledger + if err := json.Unmarshal(raw, &ledger); err != nil { + return nil, fmt.Errorf("decode stellar ledger %d failed: %w", sequence, err) + } + return &ledger, nil +} + +func (c *Client) GetPaymentsByLedger(ctx context.Context, sequence uint64, cursor string, limit int) (*PaymentsPage, error) { + if limit <= 0 { + limit = 200 + } + params := map[string]string{ + "order": "asc", + "limit": strconv.Itoa(limit), + } + if strings.TrimSpace(cursor) != "" { + params["cursor"] = cursor + } + + raw, err := c.base.Do(ctx, http.MethodGet, fmt.Sprintf("/ledgers/%d/payments", sequence), nil, params) + if err != nil { + return nil, fmt.Errorf("get stellar payments for ledger %d failed: %w", sequence, err) + } + + var page PaymentsPage + if err := json.Unmarshal(raw, &page); err != nil { + return nil, fmt.Errorf("decode stellar payments for ledger %d failed: %w", sequence, err) + } + return &page, nil +} + +func (c *Client) GetOperationsByLedger(ctx context.Context, sequence uint64, cursor string, limit int) (*OperationsPage, error) { + if limit <= 0 { + limit = 200 + } + params := map[string]string{ + "order": "asc", + "limit": strconv.Itoa(limit), + } + if strings.TrimSpace(cursor) != "" { + params["cursor"] = cursor + } + + raw, err := c.base.Do(ctx, http.MethodGet, fmt.Sprintf("/ledgers/%d/operations", sequence), nil, params) + if err != nil { + return nil, fmt.Errorf("get stellar operations for ledger %d failed: %w", sequence, err) + } + + var page OperationsPage + if err := json.Unmarshal(raw, &page); err != nil { + return nil, fmt.Errorf("decode stellar operations for ledger %d failed: %w", sequence, err) + } + return &page, nil +} + +func (c *Client) GetEffectsByOperation(ctx context.Context, operationID string, cursor string, limit int) (*EffectsPage, error) { + if limit <= 0 { + limit = 200 + } + operationID = strings.TrimSpace(operationID) + if operationID == "" { + return nil, fmt.Errorf("stellar operation id is empty") + } + params := map[string]string{ + "order": "asc", + "limit": strconv.Itoa(limit), + } + if strings.TrimSpace(cursor) != "" { + params["cursor"] = cursor + } + + raw, err := c.base.Do(ctx, http.MethodGet, fmt.Sprintf("/operations/%s/effects", operationID), nil, params) + if err != nil { + return nil, fmt.Errorf("get stellar effects for operation %s failed: %w", operationID, err) + } + + var page EffectsPage + if err := json.Unmarshal(raw, &page); err != nil { + return nil, fmt.Errorf("decode stellar effects for operation %s failed: %w", operationID, err) + } + return &page, nil +} + +func (c *Client) GetTransaction(ctx context.Context, hash string) (*Transaction, error) { + hash = strings.TrimSpace(hash) + if hash == "" { + return nil, fmt.Errorf("stellar transaction hash is empty") + } + + raw, err := c.base.Do(ctx, http.MethodGet, fmt.Sprintf("/transactions/%s", hash), nil, nil) + if err != nil { + return nil, fmt.Errorf("get stellar transaction %s failed: %w", hash, err) + } + + var tx Transaction + if err := json.Unmarshal(raw, &tx); err != nil { + return nil, fmt.Errorf("decode stellar transaction %s failed: %w", hash, err) + } + return &tx, nil +} diff --git a/internal/rpc/stellar/types.go b/internal/rpc/stellar/types.go new file mode 100644 index 0000000..cf15ace --- /dev/null +++ b/internal/rpc/stellar/types.go @@ -0,0 +1,95 @@ +package stellar + +type Root struct { + HistoryLatestLedger uint64 `json:"history_latest_ledger"` +} + +type Ledger struct { + Hash string `json:"hash"` + PrevHash string `json:"prev_hash"` + Sequence uint64 `json:"sequence"` + ClosedAt string `json:"closed_at"` + SuccessfulTxs int `json:"successful_transaction_count"` +} + +type PaymentsPage struct { + Embedded struct { + Records []Payment `json:"records"` + } `json:"_embedded"` +} + +type OperationsPage struct { + Embedded struct { + Records []Operation `json:"records"` + } `json:"_embedded"` +} + +type EffectsPage struct { + Embedded struct { + Records []Effect `json:"records"` + } `json:"_embedded"` +} + +type Payment struct { + ID string `json:"id"` + PagingToken string `json:"paging_token"` + Type string `json:"type"` + TransactionHash string `json:"transaction_hash"` + TransactionSuccessful bool `json:"transaction_successful"` + From string `json:"from"` + To string `json:"to"` + Funder string `json:"funder"` + Account string `json:"account"` + Into string `json:"into"` + SourceAccount string `json:"source_account"` + SourceAmount string `json:"source_amount"` + SourceAssetType string `json:"source_asset_type"` + SourceAssetCode string `json:"source_asset_code"` + SourceAssetIssuer string `json:"source_asset_issuer"` + Amount string `json:"amount"` + StartingBalance string `json:"starting_balance"` + AssetType string `json:"asset_type"` + AssetCode string `json:"asset_code"` + AssetIssuer string `json:"asset_issuer"` + CreatedAt string `json:"created_at"` +} + +type Transaction struct { + Hash string `json:"hash"` + Successful bool `json:"successful"` + FeeCharged string `json:"fee_charged"` + Memo string `json:"memo"` + MemoType string `json:"memo_type"` + Ledger uint64 `json:"ledger"` + CreatedAt string `json:"created_at"` +} + +type Operation struct { + ID string `json:"id"` + PagingToken string `json:"paging_token"` + Type string `json:"type"` + TransactionHash string `json:"transaction_hash"` + TransactionSuccessful bool `json:"transaction_successful"` + SourceAccount string `json:"source_account"` + CreatedAt string `json:"created_at"` + Asset string `json:"asset,omitempty"` + Amount string `json:"amount,omitempty"` + From string `json:"from,omitempty"` + BalanceID string `json:"balance_id,omitempty"` + Claimant string `json:"claimant,omitempty"` + Claimants []Claimant `json:"claimants,omitempty"` +} + +type Claimant struct { + Destination string `json:"destination"` +} + +type Effect struct { + ID string `json:"id"` + PagingToken string `json:"paging_token"` + Type string `json:"type"` + Account string `json:"account,omitempty"` + Asset string `json:"asset,omitempty"` + Amount string `json:"amount,omitempty"` + BalanceID string `json:"balance_id,omitempty"` +} diff --git a/internal/rpc/types.go b/internal/rpc/types.go index e9916c5..d54e8e3 100644 --- a/internal/rpc/types.go +++ b/internal/rpc/types.go @@ -22,6 +22,8 @@ const ( NetworkTron = "tron" // Tron blockchain NetworkBitcoin = "bitcoin" // Bitcoin blockchain NetworkCosmos = "cosmos" // Cosmos SDK / CometBFT chains + NetworkXRP = "xrp" // XRP Ledger + NetworkStellar = "stellar" // Stellar Horizon NetworkGeneric = "generic" // Generic/unknown blockchain type ) diff --git a/internal/rpc/xrp/api.go b/internal/rpc/xrp/api.go new file mode 100644 index 0000000..c40e30c --- /dev/null +++ b/internal/rpc/xrp/api.go @@ -0,0 +1,15 @@ +package xrp + +import ( + "context" + + "github.com/fystack/multichain-indexer/internal/rpc" +) + +type XRPLAPI interface { + rpc.NetworkClient + + GetLatestLedgerIndex(ctx context.Context) (uint64, error) + GetLedgerByIndex(ctx context.Context, ledgerIndex uint64) (*Ledger, error) + BatchGetLedgersByIndex(ctx context.Context, ledgerIndexes []uint64) (map[uint64]*Ledger, error) +} diff --git a/internal/rpc/xrp/client.go b/internal/rpc/xrp/client.go new file mode 100644 index 0000000..7f13155 --- /dev/null +++ b/internal/rpc/xrp/client.go @@ -0,0 +1,142 @@ +package xrp + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/fystack/multichain-indexer/internal/rpc" + "github.com/fystack/multichain-indexer/pkg/ratelimiter" +) + +type Client struct { + *rpc.BaseClient +} + +func NewClient( + url string, + auth *rpc.AuthConfig, + timeout time.Duration, + rateLimiter *ratelimiter.PooledRateLimiter, +) *Client { + return &Client{ + BaseClient: rpc.NewBaseClient( + url, + rpc.NetworkXRP, + rpc.ClientTypeRPC, + auth, + timeout, + rateLimiter, + ), + } +} + +func (c *Client) GetLatestLedgerIndex(ctx context.Context) (uint64, error) { + // Use validated ledger head (server_info), not ledger_current (open/in-progress ledger). + resp, err := c.CallRPC(ctx, "server_info", []any{map[string]any{}}) + if err != nil { + return 0, fmt.Errorf("server_info failed: %w", err) + } + + var result struct { + Info struct { + ValidatedLedger struct { + Seq uint64 `json:"seq"` + } `json:"validated_ledger"` + } `json:"info"` + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return 0, fmt.Errorf("decode server_info result: %w", err) + } + if result.Info.ValidatedLedger.Seq == 0 { + return 0, fmt.Errorf("server_info returned empty validated_ledger.seq") + } + return result.Info.ValidatedLedger.Seq, nil +} + +func (c *Client) GetLedgerByIndex(ctx context.Context, ledgerIndex uint64) (*Ledger, error) { + resp, err := c.CallRPC(ctx, "ledger", []any{map[string]any{ + "ledger_index": ledgerIndex, + "transactions": true, + "expand": true, + }}) + if err != nil { + return nil, fmt.Errorf("ledger %d failed: %w", ledgerIndex, err) + } + + var result ledgerResponse + if err := json.Unmarshal(resp.Result, &result); err != nil { + return nil, fmt.Errorf("decode ledger %d result: %w", ledgerIndex, err) + } + if result.Error != "" { + if result.ErrorMessage != "" { + return nil, fmt.Errorf("ledger %d failed: %s: %s", ledgerIndex, result.Error, result.ErrorMessage) + } + return nil, fmt.Errorf("ledger %d failed: %s", ledgerIndex, result.Error) + } + if result.Ledger == nil { + return nil, fmt.Errorf("ledger %d not found", ledgerIndex) + } + return result.Ledger, nil +} + +func (c *Client) BatchGetLedgersByIndex(ctx context.Context, ledgerIndexes []uint64) (map[uint64]*Ledger, error) { + if len(ledgerIndexes) == 0 { + return map[uint64]*Ledger{}, nil + } + + requests := make([]*rpc.RPCRequest, 0, len(ledgerIndexes)) + idToLedger := make(map[int64]uint64, len(ledgerIndexes)) + for _, ledgerIndex := range ledgerIndexes { + id := c.NextRequestIDs(1)[0] + idToLedger[id] = ledgerIndex + requests = append(requests, &rpc.RPCRequest{ + ID: id, + JSONRPC: "2.0", + Method: "ledger", + Params: []any{map[string]any{ + "ledger_index": ledgerIndex, + "transactions": true, + "expand": true, + }}, + }) + } + + responses, err := c.DoBatch(ctx, requests) + if err != nil { + return nil, fmt.Errorf("batch ledger fetch failed: %w", err) + } + + results := make(map[uint64]*Ledger, len(ledgerIndexes)) + for _, response := range responses { + id, ok := response.IDInt64() + if !ok { + return nil, fmt.Errorf("ledger batch returned non-integer id: %v", response.ID) + } + ledgerIndex, ok := idToLedger[id] + if !ok { + return nil, fmt.Errorf("ledger batch returned unknown id: %d", id) + } + if response.Error != nil { + return nil, fmt.Errorf("ledger %d failed: %w", ledgerIndex, response.Error) + } + + var result ledgerResponse + if err := json.Unmarshal(response.Result, &result); err != nil { + return nil, fmt.Errorf("decode ledger %d result: %w", ledgerIndex, err) + } + if result.Error != "" { + if result.ErrorMessage != "" { + return nil, fmt.Errorf("ledger %d failed: %s: %s", ledgerIndex, result.Error, result.ErrorMessage) + } + return nil, fmt.Errorf("ledger %d failed: %s", ledgerIndex, result.Error) + } + if result.Ledger == nil { + return nil, fmt.Errorf("ledger %d not found", ledgerIndex) + } + results[ledgerIndex] = result.Ledger + } + + return results, nil +} diff --git a/internal/rpc/xrp/types.go b/internal/rpc/xrp/types.go new file mode 100644 index 0000000..3c8a54c --- /dev/null +++ b/internal/rpc/xrp/types.go @@ -0,0 +1,74 @@ +package xrp + +import "encoding/json" + +type Ledger struct { + LedgerHash string `json:"ledger_hash"` + ParentHash string `json:"parent_hash"` + LedgerIndex json.Number `json:"ledger_index"` + CloseTime int64 `json:"close_time"` + Transactions []Transaction `json:"transactions"` +} + +type Transaction struct { + Hash string `json:"hash"` + Account string `json:"Account"` + Destination string `json:"Destination"` + DestinationTag *uint32 `json:"DestinationTag,omitempty"` + CheckID string `json:"CheckID,omitempty"` + Owner string `json:"Owner,omitempty"` + OfferSequence *uint32 `json:"OfferSequence,omitempty"` + Holder string `json:"Holder,omitempty"` + TransactionType string `json:"TransactionType"` + Fee string `json:"Fee"` + Date *int64 `json:"date,omitempty"` + LedgerIndex json.Number `json:"ledger_index"` + Amount any `json:"Amount"` + SendMax any `json:"SendMax,omitempty"` + Meta *Meta `json:"meta,omitempty"` + MetaData *Meta `json:"metaData,omitempty"` +} + +type Meta struct { + TransactionResult string `json:"TransactionResult"` + DeliveredAmount any `json:"delivered_amount"` + AffectedNodes []Node `json:"AffectedNodes"` +} + +type IssuedCurrencyAmount struct { + Currency string `json:"currency"` + Issuer string `json:"issuer"` + Value string `json:"value"` +} + +type Node struct { + CreatedNode *LedgerNode `json:"CreatedNode,omitempty"` + ModifiedNode *LedgerNode `json:"ModifiedNode,omitempty"` + DeletedNode *LedgerNode `json:"DeletedNode,omitempty"` +} + +type LedgerNode struct { + LedgerEntryType string `json:"LedgerEntryType"` + LedgerIndex string `json:"LedgerIndex"` + FinalFields *LedgerFields `json:"FinalFields,omitempty"` + NewFields *LedgerFields `json:"NewFields,omitempty"` + PreviousFields *LedgerFields `json:"PreviousFields,omitempty"` +} + +type LedgerFields struct { + Account string `json:"Account,omitempty"` + Balance any `json:"Balance,omitempty"` + Destination string `json:"Destination,omitempty"` + DestinationTag *uint32 `json:"DestinationTag,omitempty"` + Amount any `json:"Amount,omitempty"` + HighLimit any `json:"HighLimit,omitempty"` + LowLimit any `json:"LowLimit,omitempty"` +} + +type ledgerResponse struct { + Ledger *Ledger `json:"ledger"` + LedgerIndex json.Number `json:"ledger_index"` + Validated bool `json:"validated"` + Error string `json:"error"` + ErrorMessage string `json:"error_message"` +} diff --git a/internal/worker/base.go b/internal/worker/base.go index 546d0e4..a36fb8a 100644 --- a/internal/worker/base.go +++ b/internal/worker/base.go @@ -2,6 +2,7 @@ package worker import ( "context" + "maps" "strings" "time" @@ -218,15 +219,16 @@ func (bw *BaseWorker) emitBlock(block *types.Block) { if toMonitored { inTx := tx inTx.Direction = types.DirectionIn + inTx = normalizeTransactionForDirection(bw.chain, inTx, types.DirectionIn) bw.logger.Info("Emitting matched transaction", "direction", types.DirectionIn, - "from", tx.FromAddress, - "to", tx.ToAddress, + "from", inTx.FromAddress, + "to", inTx.ToAddress, "chain", bw.chain.GetName(), - "type", tx.Type, - "txhash", tx.TxHash, - "status", tx.Status, - "confirmations", tx.Confirmations, + "type", inTx.Type, + "txhash", inTx.TxHash, + "status", inTx.Status, + "confirmations", inTx.Confirmations, ) _ = bw.emitter.EmitTransaction(bw.chain.GetName(), &inTx) } @@ -234,15 +236,16 @@ func (bw *BaseWorker) emitBlock(block *types.Block) { if fromMonitored { outTx := tx outTx.Direction = types.DirectionOut + outTx = normalizeTransactionForDirection(bw.chain, outTx, types.DirectionOut) bw.logger.Info("Emitting matched transaction", "direction", types.DirectionOut, - "from", tx.FromAddress, - "to", tx.ToAddress, + "from", outTx.FromAddress, + "to", outTx.ToAddress, "chain", bw.chain.GetName(), - "type", tx.Type, - "txhash", tx.TxHash, - "status", tx.Status, - "confirmations", tx.Confirmations, + "type", outTx.Type, + "txhash", outTx.TxHash, + "status", outTx.Status, + "confirmations", outTx.Confirmations, ) _ = bw.emitter.EmitTransaction(bw.chain.GetName(), &outTx) } @@ -251,6 +254,29 @@ func (bw *BaseWorker) emitBlock(block *types.Block) { bw.emitUTXOs(block) } +func normalizeTransactionForDirection( + chain indexer.Indexer, + tx types.Transaction, + direction string, +) types.Transaction { + tx = cloneTransactionMetadata(tx) + if normalizer, ok := chain.(indexer.DirectionalNormalizer); ok { + return normalizer.NormalizeForDirection(tx, direction) + } + return tx +} + +func cloneTransactionMetadata(tx types.Transaction) types.Transaction { + if len(tx.Metadata) == 0 { + return tx + } + + cloned := make(map[string]any, len(tx.Metadata)) + maps.Copy(cloned, tx.Metadata) + tx.Metadata = cloned + return tx +} + // emitUTXOs emits UTXO events for monitored addresses. func (bw *BaseWorker) emitUTXOs(block *types.Block) { if block == nil || bw.pubkeyStore == nil { diff --git a/internal/worker/base_test.go b/internal/worker/base_test.go new file mode 100644 index 0000000..ce52aa3 --- /dev/null +++ b/internal/worker/base_test.go @@ -0,0 +1,226 @@ +package worker + +import ( + "context" + "errors" + "io" + "log/slog" + "testing" + + "github.com/fystack/multichain-indexer/internal/indexer" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/enum" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/fystack/multichain-indexer/pkg/events" +) + +const ( + testSourceAmountKey = "test_source_amount" + testSourceAssetKey = "test_source_asset" + testSourceTypeKey = "test_source_type" +) + +type capturePubkeyStore struct { + addresses map[string]struct{} +} + +func (m capturePubkeyStore) Exist(_ enum.NetworkType, address string) bool { + _, ok := m.addresses[address] + return ok +} + +func (m capturePubkeyStore) Save(enum.NetworkType, string) error { return nil } +func (m capturePubkeyStore) Close() error { return nil } + +type captureEmitter struct { + txs []types.Transaction +} + +func (e *captureEmitter) EmitBlock(string, *types.Block) error { return nil } +func (e *captureEmitter) EmitUTXO(string, *types.UTXOEvent) error { return nil } +func (e *captureEmitter) EmitError(string, error) error { return nil } +func (e *captureEmitter) Emit(events.IndexerEvent) error { return nil } +func (e *captureEmitter) Close() {} +func (e *captureEmitter) EmitTransaction(_ string, tx *types.Transaction) error { + if tx == nil { + return errors.New("nil tx") + } + e.txs = append(e.txs, *tx) + return nil +} + +type directionalTestIndexer struct{} + +var _ indexer.Indexer = (*directionalTestIndexer)(nil) +var _ indexer.DirectionalNormalizer = (*directionalTestIndexer)(nil) + +func (directionalTestIndexer) GetName() string { return "DIRECTIONAL_TEST" } +func (directionalTestIndexer) GetNetworkType() enum.NetworkType { return enum.NetworkTypeStellar } +func (directionalTestIndexer) GetNetworkInternalCode() string { return "directional_test" } +func (directionalTestIndexer) GetLatestBlockNumber(context.Context) (uint64, error) { + return 0, nil +} +func (directionalTestIndexer) GetBlock(context.Context, uint64) (*types.Block, error) { + return nil, nil +} +func (directionalTestIndexer) GetBlocks(context.Context, uint64, uint64, bool) ([]indexer.BlockResult, error) { + return nil, nil +} +func (directionalTestIndexer) GetBlocksByNumbers(context.Context, []uint64) ([]indexer.BlockResult, error) { + return nil, nil +} +func (directionalTestIndexer) IsHealthy() bool { return true } + +func (directionalTestIndexer) NormalizeForDirection(tx types.Transaction, direction string) types.Transaction { + if direction == types.DirectionOut { + if sourceType := tx.GetMetadataString(testSourceTypeKey); sourceType != "" { + tx.Type = constant.TxType(sourceType) + } + if sourceAmount := tx.GetMetadataString(testSourceAmountKey); sourceAmount != "" { + tx.Amount = sourceAmount + } + switch tx.Type { + case constant.TxTypeNativeTransfer: + tx.AssetAddress = "" + case constant.TxTypeTokenTransfer: + tx.AssetAddress = tx.GetMetadataString(testSourceAssetKey) + } + } + + if tx.Metadata != nil { + delete(tx.Metadata, testSourceTypeKey) + delete(tx.Metadata, testSourceAmountKey) + delete(tx.Metadata, testSourceAssetKey) + if len(tx.Metadata) == 0 { + tx.Metadata = nil + } + } + return tx +} + +func TestBaseWorkerEmitBlock_AppliesTokenOutgoingOverrides(t *testing.T) { + t.Parallel() + + emitter := &captureEmitter{} + bw := &BaseWorker{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + config: config.ChainConfig{TwoWayIndexing: true}, + chain: directionalTestIndexer{}, + pubkeyStore: capturePubkeyStore{addresses: map[string]struct{}{ + "GSOURCE": {}, + "GDEST": {}, + }}, + emitter: emitter, + } + + block := &types.Block{ + Transactions: []types.Transaction{ + { + TxHash: "tx-path", + NetworkId: "stellar_mainnet", + BlockHash: "block-hash", + TransferIndex: "payment-1", + FromAddress: "GSOURCE", + ToAddress: "GDEST", + Amount: "5.0000000", + Type: constant.TxTypeNativeTransfer, + Memo: "invoice-42", + Metadata: map[string]any{ + testSourceTypeKey: string(constant.TxTypeTokenTransfer), + testSourceAmountKey: "10.0000000", + testSourceAssetKey: "GISSUER:USDC", + }, + }, + }, + } + + bw.emitBlock(block) + if len(emitter.txs) != 2 { + t.Fatalf("expected 2 emitted txs, got %d", len(emitter.txs)) + } + + inTx := emitter.txs[0] + if inTx.Direction != types.DirectionIn { + t.Fatalf("expected incoming direction, got %s", inTx.Direction) + } + if inTx.Type != constant.TxTypeNativeTransfer || inTx.Amount != "5.0000000" || inTx.AssetAddress != "" { + t.Fatalf("incoming tx should keep receiver-side routing, got type=%s amount=%s asset=%s", inTx.Type, inTx.Amount, inTx.AssetAddress) + } + if inTx.Memo != "invoice-42" { + t.Fatalf("incoming tx should preserve unrelated metadata") + } + if inTx.GetMetadataString(testSourceTypeKey) != "" || inTx.GetMetadataString(testSourceAmountKey) != "" || inTx.GetMetadataString(testSourceAssetKey) != "" { + t.Fatalf("incoming tx should not leak directional override metadata") + } + + outTx := emitter.txs[1] + if outTx.Direction != types.DirectionOut { + t.Fatalf("expected outgoing direction, got %s", outTx.Direction) + } + if outTx.Type != constant.TxTypeTokenTransfer || outTx.Amount != "10.0000000" || outTx.AssetAddress != "GISSUER:USDC" { + t.Fatalf("outgoing tx should use source-side routing, got type=%s amount=%s asset=%s", outTx.Type, outTx.Amount, outTx.AssetAddress) + } + if outTx.Memo != "invoice-42" { + t.Fatalf("outgoing tx should preserve unrelated metadata") + } + if outTx.GetMetadataString(testSourceTypeKey) != "" || outTx.GetMetadataString(testSourceAmountKey) != "" || outTx.GetMetadataString(testSourceAssetKey) != "" { + t.Fatalf("outgoing tx should not leak directional override metadata") + } +} + +func TestBaseWorkerEmitBlock_AppliesNativeOutgoingOverrides(t *testing.T) { + t.Parallel() + + emitter := &captureEmitter{} + bw := &BaseWorker{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + config: config.ChainConfig{TwoWayIndexing: true}, + chain: directionalTestIndexer{}, + pubkeyStore: capturePubkeyStore{addresses: map[string]struct{}{ + "GSOURCE": {}, + "GDEST": {}, + }}, + emitter: emitter, + } + + block := &types.Block{ + Transactions: []types.Transaction{ + { + TxHash: "tx-path", + NetworkId: "stellar_mainnet", + BlockHash: "block-hash", + TransferIndex: "payment-2", + FromAddress: "GSOURCE", + ToAddress: "GDEST", + Amount: "5.0000000", + Type: constant.TxTypeTokenTransfer, + AssetAddress: "GISSUER:USDC", + MemoType: types.MemoTypeText, + Metadata: map[string]any{ + testSourceTypeKey: string(constant.TxTypeNativeTransfer), + testSourceAmountKey: "7.5000000", + }, + }, + }, + } + + bw.emitBlock(block) + if len(emitter.txs) != 2 { + t.Fatalf("expected 2 emitted txs, got %d", len(emitter.txs)) + } + + outTx := emitter.txs[1] + if outTx.Direction != types.DirectionOut { + t.Fatalf("expected outgoing direction, got %s", outTx.Direction) + } + if outTx.Type != constant.TxTypeNativeTransfer || outTx.Amount != "7.5000000" || outTx.AssetAddress != "" { + t.Fatalf("outgoing native override should clear receiver-side asset, got type=%s amount=%s asset=%s", outTx.Type, outTx.Amount, outTx.AssetAddress) + } + if outTx.MemoType != types.MemoTypeText { + t.Fatalf("outgoing tx should preserve unrelated metadata") + } + if outTx.GetMetadataString(testSourceTypeKey) != "" || outTx.GetMetadataString(testSourceAmountKey) != "" || outTx.GetMetadataString(testSourceAssetKey) != "" { + t.Fatalf("outgoing tx should not leak directional override metadata") + } +} diff --git a/internal/worker/factory.go b/internal/worker/factory.go index 6d14611..e5d0119 100644 --- a/internal/worker/factory.go +++ b/internal/worker/factory.go @@ -15,10 +15,12 @@ import ( "github.com/fystack/multichain-indexer/internal/rpc/cosmos" "github.com/fystack/multichain-indexer/internal/rpc/evm" "github.com/fystack/multichain-indexer/internal/rpc/solana" + "github.com/fystack/multichain-indexer/internal/rpc/stellar" "github.com/fystack/multichain-indexer/internal/rpc/sui" tonrpc "github.com/fystack/multichain-indexer/internal/rpc/ton" "github.com/fystack/multichain-indexer/internal/rpc/tron" "github.com/fystack/multichain-indexer/internal/status" + "github.com/fystack/multichain-indexer/internal/rpc/xrp" "github.com/fystack/multichain-indexer/pkg/addressbloomfilter" "github.com/fystack/multichain-indexer/pkg/common/config" "github.com/fystack/multichain-indexer/pkg/common/enum" @@ -488,6 +490,89 @@ func buildTonIndexer( ) } +// buildXRPIndexer constructs an XRP indexer with failover and providers. +func buildXRPIndexer( + chainName string, + chainCfg config.ChainConfig, + mode WorkerMode, + pubkeyStore pubkeystore.Store, +) indexer.Indexer { + failover := rpc.NewFailover[xrp.XRPLAPI](nil) + + rl := ratelimiter.GetOrCreateSharedPooledRateLimiter( + chainName, chainCfg.Throttle.RPS, chainCfg.Throttle.Burst, + ) + + for i, node := range chainCfg.Nodes { + client := xrp.NewClient( + node.URL, + &rpc.AuthConfig{ + Type: rpc.AuthType(node.Auth.Type), + Key: node.Auth.Key, + Value: node.Auth.Value, + }, + chainCfg.Client.Timeout, + rl, + ) + if len(node.Headers) > 0 { + client.SetCustomHeaders(node.Headers) + } + + failover.AddProvider(&rpc.Provider{ + Name: chainName + "-" + strconv.Itoa(i+1), + URL: node.URL, + Network: chainName, + ClientType: rpc.ClientTypeRPC, + Client: client, + State: rpc.StateHealthy, + }) + } + + return indexer.NewXRPIndexer(chainName, chainCfg, failover, pubkeyStore) +} + +// buildStellarIndexer constructs a Stellar indexer with failover and providers. +func buildStellarIndexer( + chainName string, + chainCfg config.ChainConfig, + mode WorkerMode, + kvstore infra.KVStore, + pubkeyStore pubkeystore.Store, +) indexer.Indexer { + failover := rpc.NewFailover[stellar.StellarAPI](nil) + + rl := ratelimiter.GetOrCreateSharedPooledRateLimiter( + chainName, chainCfg.Throttle.RPS, chainCfg.Throttle.Burst, + ) + + for i, node := range chainCfg.Nodes { + client := stellar.NewClient( + node.URL, + &rpc.AuthConfig{ + Type: rpc.AuthType(node.Auth.Type), + Key: node.Auth.Key, + Value: node.Auth.Value, + }, + chainCfg.Client.Timeout, + rl, + ) + if len(node.Headers) > 0 { + client.SetCustomHeaders(node.Headers) + } + + failover.AddProvider(&rpc.Provider{ + Name: chainName + "-" + strconv.Itoa(i+1), + URL: node.URL, + Network: chainName, + ClientType: rpc.ClientTypeREST, + Client: client, + State: rpc.StateHealthy, + }) + } + + return indexer.NewStellarIndexer(chainName, chainCfg, failover, kvstore, pubkeyStore) +} + func preloadTONJettonWallets( ctx context.Context, chainName string, @@ -811,6 +896,10 @@ func CreateManagerWithWorkers( idxr = buildAptosIndexer(chainName, chainCfg, ModeRegular, pubkeyStore) case enum.NetworkTypeTon: idxr = buildTonIndexer(chainName, chainCfg, pubkeyStore, db, redisClient) + case enum.NetworkTypeXRP: + idxr = buildXRPIndexer(chainName, chainCfg, ModeRegular, pubkeyStore) + case enum.NetworkTypeStellar: + idxr = buildStellarIndexer(chainName, chainCfg, ModeRegular, kvstore, pubkeyStore) default: logger.Fatal("Unsupported network type", "chain", chainName, "type", chainCfg.Type) } diff --git a/pkg/common/enum/enum.go b/pkg/common/enum/enum.go index f70ce21..e58e799 100644 --- a/pkg/common/enum/enum.go +++ b/pkg/common/enum/enum.go @@ -19,14 +19,16 @@ const ( ) const ( - NetworkTypeEVM NetworkType = "evm" - NetworkTypeTron NetworkType = "tron" - NetworkTypeBtc NetworkType = "btc" - NetworkTypeSol NetworkType = "sol" - NetworkTypeApt NetworkType = "apt" - NetworkTypeSui NetworkType = "sui" - NetworkTypeCosmos NetworkType = "cosmos" - NetworkTypeTon NetworkType = "ton" + NetworkTypeEVM NetworkType = "evm" + NetworkTypeTron NetworkType = "tron" + NetworkTypeBtc NetworkType = "btc" + NetworkTypeSol NetworkType = "sol" + NetworkTypeApt NetworkType = "apt" + NetworkTypeSui NetworkType = "sui" + NetworkTypeCosmos NetworkType = "cosmos" + NetworkTypeTon NetworkType = "ton" + NetworkTypeXRP NetworkType = "xrp" + NetworkTypeStellar NetworkType = "stellar" ) var AllNetworkTypes = []NetworkType{ @@ -38,6 +40,8 @@ var AllNetworkTypes = []NetworkType{ NetworkTypeSui, NetworkTypeCosmos, NetworkTypeTon, + NetworkTypeXRP, + NetworkTypeStellar, } const ( diff --git a/pkg/common/types/types.go b/pkg/common/types/types.go index 14d5bf7..80e15a5 100644 --- a/pkg/common/types/types.go +++ b/pkg/common/types/types.go @@ -11,6 +11,26 @@ import ( "github.com/shopspring/decimal" ) +// MemoType represents a chain-specific memo type. +// +// For Stellar this maps to the underlying XDR memo type: +// - "none" -> MEMO_NONE +// - "text" -> MEMO_TEXT +// - "id" -> MEMO_ID +// - "hash" -> MEMO_HASH +// - "return" -> MEMO_RETURN (32-byte refund hash) +// +// Other chains may reuse these values or leave MemoType empty. +type MemoType string + +const ( + MemoTypeNone MemoType = "none" + MemoTypeText MemoType = "text" + MemoTypeID MemoType = "id" + MemoTypeHash MemoType = "hash" + MemoTypeReturn MemoType = "return" +) + type Block struct { Number uint64 `json:"number"` Hash string `json:"hash"` @@ -42,23 +62,26 @@ const ( ) type Transaction struct { - TxHash string `json:"txHash"` - NetworkId string `json:"networkId"` - BlockNumber uint64 `json:"blockNumber"` // 0 for mempool transactions - BlockHash string `json:"blockHash"` // block hash for reorg-aware idempotency - TransferIndex string `json:"transferIndex"` // unique position within tx (EVM only) - FromAddress string `json:"fromAddress"` - FromAddresses []string `json:"fromAddresses,omitempty"` - ToAddress string `json:"toAddress"` - AssetAddress string `json:"assetAddress"` - Amount string `json:"amount"` - Type constant.TxType `json:"type"` - TxFee decimal.Decimal `json:"txFee"` - Timestamp uint64 `json:"timestamp"` - Confirmations uint64 `json:"confirmations"` // Number of confirmations (0 = mempool/unconfirmed) - Status string `json:"status"` // "pending" (0 conf), "confirmed" (1+ conf) - Direction string `json:"direction"` // "in" (deposit) or "out" (withdrawal) - Metadata map[string]any `json:"metadata,omitempty"` + TxHash string `json:"txHash"` + NetworkId string `json:"networkId"` + BlockNumber uint64 `json:"blockNumber"` // 0 for mempool transactions + BlockHash string `json:"blockHash"` // block hash for reorg-aware idempotency + TransferIndex string `json:"transferIndex"` // stable position or operation id when the chain can provide one + FromAddress string `json:"fromAddress"` + FromAddresses []string `json:"fromAddresses,omitempty"` + ToAddress string `json:"toAddress"` + AssetAddress string `json:"assetAddress"` + Amount string `json:"amount"` + Type constant.TxType `json:"type"` + TxFee decimal.Decimal `json:"txFee"` + Timestamp uint64 `json:"timestamp"` + Confirmations uint64 `json:"confirmations"` // Number of confirmations (0 = mempool/unconfirmed) + Status string `json:"status"` // "pending" (0 conf), "confirmed" (1+ conf) + Direction string `json:"direction"` // "in" (deposit) or "out" (withdrawal) + DestinationTag string `json:"destinationTag,omitempty"` + Memo string `json:"memo,omitempty"` + MemoType MemoType `json:"memoType,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` } func (t *Transaction) SetMetadata(key string, value any) { @@ -105,6 +128,18 @@ func (t Transaction) AllSenderAddresses() []string { return nil } +const ( + MetadataKeyDestinationTag = "destination_tag" + MetadataKeyMemo = "memo" + MetadataKeyMemoType = "memo_type" + MetadataKeySubtype = "subtype" + MetadataKeyCheckID = "check_id" + MetadataKeyEscrowOwner = "escrow_owner" + MetadataKeyEscrowSequence = "escrow_sequence" + MetadataKeyClaimableID = "claimable_balance_id" + MetadataKeyClaimants = "claimant_addresses" +) + func (t Transaction) MarshalBinary() ([]byte, error) { bytes, err := json.Marshal(t) if err != nil { @@ -138,7 +173,7 @@ func (t Transaction) String() string { // Hash generates a deterministic hash used as the NATS idempotency key (Event Instance Identity). // -// When TransferIndex is set (EVM): NetworkId|TxHash|BlockHash|TransferIndex|Direction +// When TransferIndex is set: NetworkId|TxHash|BlockHash|TransferIndex|Direction // When TransferIndex is empty (non-EVM fallback): NetworkId|TxHash|BlockHash|From|To|Timestamp|Direction // // BlockHash ensures reorgs produce new hashes so consumers get updated data. @@ -153,8 +188,8 @@ func (t Transaction) Hash() string { builder.WriteByte('|') if t.TransferIndex != "" { - // EVM (or any chain that populates TransferIndex): - // Exact positional identity — reorg-aware via BlockHash. + // Chains that populate TransferIndex get exact positional identity, + // still reorg-aware via BlockHash. builder.WriteString(t.TransferIndex) builder.WriteByte('|') builder.WriteString(t.Direction) @@ -169,6 +204,18 @@ func (t Transaction) Hash() string { builder.WriteString(strconv.FormatUint(t.Timestamp, 10)) builder.WriteByte('|') builder.WriteString(t.Direction) + if destinationTag := strings.TrimSpace(t.DestinationTag); destinationTag != "" { + builder.WriteByte('|') + builder.WriteString(destinationTag) + } + if memo := strings.TrimSpace(t.Memo); memo != "" { + builder.WriteByte('|') + builder.WriteString(memo) + } + if memoType := strings.TrimSpace(string(t.MemoType)); memoType != "" { + builder.WriteByte('|') + builder.WriteString(memoType) + } } hash := sha256.Sum256([]byte(builder.String())) diff --git a/pkg/common/types/types_test.go b/pkg/common/types/types_test.go new file mode 100644 index 0000000..fa3f326 --- /dev/null +++ b/pkg/common/types/types_test.go @@ -0,0 +1,66 @@ +package types + +import ( + "testing" + + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/shopspring/decimal" + "github.com/stretchr/testify/require" +) + +func TestTransactionBinaryRoundTripPreservesRoutingFields(t *testing.T) { + t.Parallel() + + tx := Transaction{ + TxHash: "tx-hash", + NetworkId: "xrp_mainnet", + BlockNumber: 42, + FromAddress: "rFrom", + ToAddress: "rTo", + AssetAddress: "", + Amount: "10.5", + Type: constant.TxTypeNativeTransfer, + TxFee: decimal.RequireFromString("0.000012"), + Timestamp: 1234567890, + Confirmations: 1, + Status: StatusConfirmed, + Direction: DirectionIn, + } + tx.DestinationTag = "12345" + tx.Memo = "invoice-42" + tx.MemoType = MemoTypeText + tx.SetMetadata("custom_routing_note", "internal") + + data, err := tx.MarshalBinary() + require.NoError(t, err) + + var decoded Transaction + require.NoError(t, decoded.UnmarshalBinary(data)) + require.Equal(t, "12345", decoded.DestinationTag) + require.Equal(t, "invoice-42", decoded.Memo) + require.Equal(t, MemoTypeText, decoded.MemoType) + require.Equal(t, "internal", decoded.GetMetadataString("custom_routing_note")) +} + +func TestTransactionHashIncludesRoutingFields(t *testing.T) { + t.Parallel() + + base := Transaction{ + TxHash: "tx-hash", + NetworkId: "xrp_mainnet", + FromAddress: "rFrom", + ToAddress: "rTo", + Timestamp: 1234567890, + Direction: DirectionIn, + } + + withTag := base + withTag.DestinationTag = "1" + + withMemo := base + withMemo.Memo = "memo" + withMemo.MemoType = MemoTypeText + + require.NotEqual(t, base.Hash(), withTag.Hash()) + require.NotEqual(t, base.Hash(), withMemo.Hash()) +}