diff --git a/go.mod b/go.mod index d9604a11de..78476411ca 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/bsv-blockchain/go-bt/v2 v2.6.3 github.com/bsv-blockchain/go-chaincfg v1.5.8 github.com/bsv-blockchain/go-sdk v1.2.23 - github.com/bsv-blockchain/go-subtree v1.3.3 + github.com/bsv-blockchain/go-subtree v1.4.1 github.com/bsv-blockchain/testcontainers-aerospike-go v0.3.2 github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd github.com/btcsuite/goleveldb v1.0.0 diff --git a/go.sum b/go.sum index 5bb3cae058..e33ba89f0c 100644 --- a/go.sum +++ b/go.sum @@ -194,8 +194,8 @@ github.com/bsv-blockchain/go-safe-conversion v1.2.0 h1:HwLWaxqm2OyU5/2BRQ8kwdn25 github.com/bsv-blockchain/go-safe-conversion v1.2.0/go.mod h1:62Eq2980j4BygfqJhd/ObGRHET8ubEy2rBdGi4k6nsw= github.com/bsv-blockchain/go-sdk v1.2.23 h1:DRZWaqgW6Ra+uFe1+sLFi+WPcjTdBm8NAwfr6ODOF68= github.com/bsv-blockchain/go-sdk v1.2.23/go.mod h1:5mmw1QLusuAkjWmQgUOurQYCXdIsQEsWXbAZ9zwme3g= -github.com/bsv-blockchain/go-subtree v1.3.3 h1:q6xCvXm2lK3HCcBep24d1tjaEPIOKl5ocq7gShW6qEM= -github.com/bsv-blockchain/go-subtree v1.3.3/go.mod h1:0ysEa29As06LxqoY1+tT+oLMWbDp7Pj0DFILZ84eGoc= +github.com/bsv-blockchain/go-subtree v1.4.1 h1:9cQE3KPLP0kghZHadPbMuaRKgaxbc86wyLOH7LYfnQw= +github.com/bsv-blockchain/go-subtree v1.4.1/go.mod h1:0ysEa29As06LxqoY1+tT+oLMWbDp7Pj0DFILZ84eGoc= github.com/bsv-blockchain/go-tx-map v1.3.7 h1:59LVYW2bvax/WbyM1eUNkrojXNrVu9f0lAuKbNryEbQ= github.com/bsv-blockchain/go-tx-map v1.3.7/go.mod h1:QJo02rdqFTHaimSTCgl6mLMLwIDtykwWTC1lt7nrhY8= github.com/bsv-blockchain/go-wire v1.2.3 h1:dqnpNFgqw8ZQAlzNCe3qjv39SrYYDMsWXhVneaR2a7E= diff --git a/model/Block_test.go b/model/Block_test.go index 2b13e98e39..5f48d1c4d9 100644 --- a/model/Block_test.go +++ b/model/Block_test.go @@ -3134,12 +3134,20 @@ func createSubtreeMetadataWithParents(subtree *subtreepkg.Subtree, nodeIndex int for i := 0; i < subtree.Length(); i++ { // Add parent hashes to specific node if i == nodeIndex && len(parentHashes) > 0 { - txInpoints := subtreepkg.NewTxInpoints() - // Add parent hashes (simplified - in real usage would need proper input indices) - for _, parentHash := range parentHashes { - // Create mock input with parent hash - txInpoints.ParentTxHashes = append(txInpoints.ParentTxHashes, parentHash) - txInpoints.Idxs = append(txInpoints.Idxs, []uint32{0}) // Mock output index + // Build mock inputs — vout 0 for each parent hash. + inputs := make([]*bt.Input, 0, len(parentHashes)) + for j := range parentHashes { + in := &bt.Input{PreviousTxOutIndex: 0} + if err := in.PreviousTxIDAdd(&parentHashes[j]); err != nil { + return nil, err + } + + inputs = append(inputs, in) + } + + txInpoints, err := subtreepkg.NewTxInpointsFromInputs(inputs) + if err != nil { + return nil, err } subtreeMeta.TxInpoints[i] = txInpoints @@ -4472,26 +4480,35 @@ func TestValidateSubtreeBenchmark(t *testing.T) { // Create subtree metadata subtreeMeta := subtreepkg.NewSubtreeMeta(subtree) for i := 0; i < subtree.Length(); i++ { - txInpoints := subtreepkg.NewTxInpoints() // Skip coinbase placeholder in first subtree if s == 0 && i == 0 { - subtreeMeta.TxInpoints[i] = txInpoints + subtreeMeta.TxInpoints[i] = subtreepkg.NewTxInpoints() continue } + var parentHash *chainhash.Hash if i <= numExternalParents { // First N transactions reference external parents (need UTXO lookup) - txInpoints.ParentTxHashes = append(txInpoints.ParentTxHashes, allParentHashes[s][i]) - txInpoints.Idxs = append(txInpoints.Idxs, []uint32{0}) + parentHash = &allParentHashes[s][i] } else { // Remaining transactions reference the previous tx in the subtree (in txMap) prevIdx := i - 1 if prevIdx >= 0 && prevIdx < len(allTxHashes[s]) { - txInpoints.ParentTxHashes = append(txInpoints.ParentTxHashes, allTxHashes[s][prevIdx]) - txInpoints.Idxs = append(txInpoints.Idxs, []uint32{0}) + parentHash = &allTxHashes[s][prevIdx] } } - subtreeMeta.TxInpoints[i] = txInpoints + + if parentHash != nil { + in := &bt.Input{PreviousTxOutIndex: 0} + require.NoError(t, in.PreviousTxIDAdd(parentHash)) + + ti, err := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{in}) + require.NoError(t, err) + + subtreeMeta.TxInpoints[i] = ti + } else { + subtreeMeta.TxInpoints[i] = subtreepkg.NewTxInpoints() + } } subtreeMetaBytes, err := subtreeMeta.Serialize() diff --git a/model/TestHelper.go b/model/TestHelper.go index 2058340563..aaf44593bd 100644 --- a/model/TestHelper.go +++ b/model/TestHelper.go @@ -131,10 +131,17 @@ func GenerateTestBlock(transactionIDCount uint64, subtreeStore *TestLocalSubtree binary.LittleEndian.PutUint64(parentTxID[:], uint64(i-1)) // Reference the previous transaction as parent parentHash := chainhash.Hash(parentTxID) - if err = subtreeMeta.SetTxInpoints(len(subtree.Nodes)-1, subtreepkg.TxInpoints{ - ParentTxHashes: []chainhash.Hash{parentHash}, - Idxs: [][]uint32{{0}}, // Reference output 0 of parent transaction - }); err != nil { + parentInput := &bt.Input{PreviousTxOutIndex: 0} + if err = parentInput.PreviousTxIDAdd(&parentHash); err != nil { + return nil, err + } + + txInpoints, err := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{parentInput}) + if err != nil { + return nil, err + } + + if err = subtreeMeta.SetTxInpoints(len(subtree.Nodes)-1, txInpoints); err != nil { return nil, err } diff --git a/services/asset/httpimpl/GetTransactionMeta_test.go b/services/asset/httpimpl/GetTransactionMeta_test.go index 83a372e0fb..848e61969a 100644 --- a/services/asset/httpimpl/GetTransactionMeta_test.go +++ b/services/asset/httpimpl/GetTransactionMeta_test.go @@ -5,6 +5,7 @@ import ( "net/http" "testing" + "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/go-subtree" "github.com/bsv-blockchain/teranode/errors" @@ -16,10 +17,22 @@ import ( "github.com/stretchr/testify/require" ) -var ( - transactionMeta = &meta.Data{ +func newTransactionMeta() *meta.Data { + parent := testBlockHeader.Hash() + + in := &bt.Input{PreviousTxOutIndex: 1} + if err := in.PreviousTxIDAdd(parent); err != nil { + panic(err) + } + + ti, err := subtree.NewTxInpointsFromInputs([]*bt.Input{in}) + if err != nil { + panic(err) + } + + return &meta.Data{ Tx: nil, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*testBlockHeader.Hash()}, Idxs: [][]uint32{{1}}}, + TxInpoints: ti, BlockIDs: []uint32{1, 2, 3}, SubtreeIdxs: []int{0, 0, 0}, // Add subtree indices Fee: 123, @@ -27,7 +40,9 @@ var ( IsCoinbase: false, LockTime: 500000, } -) +} + +var transactionMeta = newTransactionMeta() func TestGetTransactionMeta(t *testing.T) { initPrometheusMetrics() diff --git a/services/asset/httpimpl/test.go b/services/asset/httpimpl/test.go index d08dca085d..f409d170f8 100644 --- a/services/asset/httpimpl/test.go +++ b/services/asset/httpimpl/test.go @@ -71,11 +71,12 @@ var ( Height: 100, ID: 666, } - testBlockBytes, _ = testBlock.Bytes() - testSubtree, _ = subtree.NewTreeByLeafCount(4) - testTxMeta = &meta.Data{ + testBlockBytes, _ = testBlock.Bytes() + testSubtree, _ = subtree.NewTreeByLeafCount(4) + testTxMetaInpoints, _ = subtree.NewTxInpointsFromInputs(testTx1.Inputs) + testTxMeta = &meta.Data{ Tx: testTx1, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*testTx1.Inputs[0].PreviousTxIDChainHash()}, Idxs: [][]uint32{{testTx1.Inputs[0].PreviousTxOutIndex}}}, + TxInpoints: testTxMetaInpoints, BlockIDs: []uint32{100}, Fee: 123, SizeInBytes: 321, diff --git a/services/blockassembly/Client.go b/services/blockassembly/Client.go index bd9e5ddfad..4bdab7e5b5 100644 --- a/services/blockassembly/Client.go +++ b/services/blockassembly/Client.go @@ -478,15 +478,21 @@ func (s *Client) sendBatchColumnar(ctx context.Context, batch []*batchItem) { // and sends them in columnar format. This shifts deserialization work away from the single-machine Server. // // The columnar format packs all data into contiguous arrays: -// - txids_packed: All 32-byte TXIDs concatenated -// - fees: All fees in a single array -// - sizes: All sizes in a single array -// - parent_tx_hashes_packed: All parent tx hashes (from TxInpoints) concatenated -// - parent_tx_offsets: Offset table for parent hashes per transaction -// - parent_vout_indices: All vout indices flattened -// - vout_idx_offsets: Offset table for vout indices per parent hash -// -// This reduces allocations and moves deserialization to distributed Clients. +// - txids_packed: All 32-byte TXIDs concatenated +// - fees: All fees in a single array +// - sizes: All sizes in a single array +// - parent_tx_hashes_packed: All parent tx hashes (from TxInpoints) concatenated +// - parent_tx_offsets: Offset table for parent hashes per transaction +// - vout_idxs_packed: Count-prefixed packed vouts in the exact shape the +// server's TxInpoints.voutIdxs stores internally. For each parent, one +// uint32 count word followed by that many vout-value words, concatenated +// across all transactions. +// - vout_idxs_tx_offsets: Per-tx offsets into vout_idxs_packed. +// +// This pushes all TxInpoints layout work to the Client (horizontally scalable +// validators) so the single-instance block-assembly Server can construct +// TxInpoints from a per-tx slice with zero allocation +// (subtree.NewTxInpointsFromPacked). func (s *Client) convertToColumnarFormat(batch []*batchItem) (*blockassembly_api.AddTxBatchColumnarRequest, error) { batchSize := len(batch) if batchSize == 0 { @@ -499,23 +505,22 @@ func (s *Client) convertToColumnarFormat(batch []*batchItem) (*blockassembly_api fees := make([]uint64, batchSize) sizes := make([]uint64, batchSize) parentTxOffsets := make([]uint32, batchSize+1) + voutIdxsTxOffsets := make([]uint32, batchSize+1) // For variable-length fields, estimate capacity based on typical usage // Estimate: avg 3 parent hashes per tx estimatedParentHashes := batchSize * 3 parentTxHashesPacked := make([]byte, 0, estimatedParentHashes*32) - // Estimate: avg 2 vout indices per parent hash - estimatedVoutIndices := estimatedParentHashes * 2 - parentVoutIndices := make([]uint32, 0, estimatedVoutIndices) - voutIdxOffsets := make([]uint32, 1, estimatedParentHashes+1) + // Estimate: avg 2 vout indices per parent hash, plus a count word per + // parent. Sized as estimatedParentHashes * 3 ≈ count word + 2 values. + voutIdxsPacked := make([]uint32, 0, estimatedParentHashes*3) // Start with offset 0 parentTxOffsets[0] = 0 - voutIdxOffsets[0] = 0 + voutIdxsTxOffsets[0] = 0 currentParentHashCount := uint32(0) - currentVoutIdxCount := uint32(0) for i, item := range batch { req := item.req @@ -544,15 +549,20 @@ func (s *Client) convertToColumnarFormat(batch []*batchItem) (*blockassembly_api } parentTxOffsets[i+1] = currentParentHashCount - // Pack vout indices (2D array flattened) - // Avoid extra allocations by using Idxs directly + // Pack vouts in count-prefixed layout, one parent at a time. The + // resulting slice is byte-identical to TxInpoints.voutIdxs on the + // Server side, so it can be aliased directly with no decoding. for j := range parentHashes { - // Idxs is [][]uint32, where Idxs[j] contains the vout indices for parentHashes[j] - vouts := txInpoints.Idxs[j] - parentVoutIndices = append(parentVoutIndices, vouts...) - currentVoutIdxCount += uint32(len(vouts)) - voutIdxOffsets = append(voutIdxOffsets, currentVoutIdxCount) + vouts, err := txInpoints.GetParentVoutsAtIndex(j) + if err != nil { + return nil, errors.NewInvalidArgumentError("failed to read vouts at parent %d of tx %d: %v", j, i, err) + } + + voutIdxsPacked = append(voutIdxsPacked, uint32(len(vouts))) + voutIdxsPacked = append(voutIdxsPacked, vouts...) } + + voutIdxsTxOffsets[i+1] = uint32(len(voutIdxsPacked)) } return &blockassembly_api.AddTxBatchColumnarRequest{ @@ -561,8 +571,8 @@ func (s *Client) convertToColumnarFormat(batch []*batchItem) (*blockassembly_api Sizes: sizes, ParentTxHashesPacked: parentTxHashesPacked, ParentTxOffsets: parentTxOffsets, - ParentVoutIndices: parentVoutIndices, - VoutIdxOffsets: voutIdxOffsets, + VoutIdxsPacked: voutIdxsPacked, + VoutIdxsTxOffsets: voutIdxsTxOffsets, }, nil } diff --git a/services/blockassembly/Server.go b/services/blockassembly/Server.go index 9918d29ea7..3a54770ec8 100644 --- a/services/blockassembly/Server.go +++ b/services/blockassembly/Server.go @@ -19,6 +19,7 @@ import ( "net/http" "sync" "time" + "unsafe" "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" @@ -979,30 +980,34 @@ func (ba *BlockAssembly) AddTxBatch(ctx context.Context, batch *blockassembly_ap return nil, errors.WrapGRPC(errors.NewInvalidArgumentError("no tx requests in batch")) } - // Build batch arrays + // Build batch arrays — three allocations for the whole batch. Per-tx + // TxInpoints values live inline in txInpointsArr so the loop only takes + // a pointer (no per-tx heap escape for the TxInpoints struct itself). nodes := make([]subtreepkg.Node, len(requests)) + txInpointsArr := make([]subtreepkg.TxInpoints, len(requests)) txInpointsList := make([]*subtreepkg.TxInpoints, len(requests)) - var err error + storeTxInpoints := ba.settings.BlockAssembly.StoreTxInpointsForSubtreeMeta for i, req := range requests { - var txInpoints subtreepkg.TxInpoints - if ba.settings.BlockAssembly.StoreTxInpointsForSubtreeMeta { - txInpoints, err = subtreepkg.NewTxInpointsFromBytes(req.TxInpoints) - if err != nil { - return nil, errors.WrapGRPC(errors.NewProcessingError("unable to deserialize tx inpoints", err)) - } - } else { - // Create empty TxInpoints if not storing for subtree meta - txInpoints = subtreepkg.TxInpoints{} - } - nodes[i] = subtreepkg.Node{ Hash: chainhash.Hash(req.Txid), Fee: req.Fee, SizeInBytes: req.Size, } - txInpointsList[i] = &txInpoints + + if storeTxInpoints { + ti, err := subtreepkg.NewTxInpointsFromBytes(req.TxInpoints) + if err != nil { + return nil, errors.WrapGRPC(errors.NewProcessingError("unable to deserialize tx inpoints", err)) + } + + txInpointsArr[i] = ti + } + // else: txInpointsArr[i] stays zero-valued — the empty TxInpoints + // behaviour for callers that do not need parent inpoints stored. + + txInpointsList[i] = &txInpointsArr[i] } prometheusBlockAssemblyAddTxCounter.Add(float64(len(nodes))) // gosec:nolint @@ -1077,73 +1082,117 @@ func (ba *BlockAssembly) AddTxBatchColumnar(ctx context.Context, req *blockassem return nil, errors.WrapGRPC(errors.NewInvalidArgumentError("parent_tx_hashes_packed length must be divisible by 32")) } - totalParentHashes := len(req.ParentTxHashesPacked) / 32 - if len(req.VoutIdxOffsets) != totalParentHashes+1 { - return nil, errors.WrapGRPC(errors.NewInvalidArgumentError("vout_idx_offsets must have exactly (total_parent_hashes+1) elements (got %d, expected %d)", len(req.VoutIdxOffsets), totalParentHashes+1)) + if len(req.VoutIdxsTxOffsets) != txCount+1 { + return nil, errors.WrapGRPC(errors.NewInvalidArgumentError( + "vout_idxs_tx_offsets must have exactly txCount+1 elements (got %d, expected %d)", + len(req.VoutIdxsTxOffsets), txCount+1)) } - if ba.settings.BlockAssembly.Disabled { - return &blockassembly_api.AddTxBatchResponse{Ok: true}, nil + totalParents := len(req.ParentTxHashesPacked) / 32 + voutIdxsLen := len(req.VoutIdxsPacked) + + // Validate the two offset-array endpoints + monotonicity. Without this a + // malformed request triggers a slice-bounds-out-of-range panic inside + // the per-tx loop, and grpc-go does not recover handler panics — a + // single bad packet would crash the entire block-assembly process. The + // validator is trusted at the semantic layer, so we do NOT walk the + // packed voutIdxs to verify the count-prefix invariant (that walk would + // be O(B·P) at 1M+ TPS); we only do what is needed to make every slice + // expression in the per-tx loop bounds-safe. + // + // Cost: O(txCount) comparisons, ~0.1 % of a core at 1M TPS / batch 1000. + if req.ParentTxOffsets[0] != 0 { + return nil, errors.WrapGRPC(errors.NewInvalidArgumentError( + "parent_tx_offsets[0] must be 0, got %d", req.ParentTxOffsets[0])) } - // Build batch arrays - nodes := make([]subtreepkg.Node, txCount) - txInpointsList := make([]*subtreepkg.TxInpoints, txCount) + if req.VoutIdxsTxOffsets[0] != 0 { + return nil, errors.WrapGRPC(errors.NewInvalidArgumentError( + "vout_idxs_tx_offsets[0] must be 0, got %d", req.VoutIdxsTxOffsets[0])) + } - // Process each transaction using column-oriented access - for i := 0; i < txCount; i++ { - // Extract TXID (32 bytes) - no allocation, just slice reference - txidStart := i * 32 - txid := req.TxidsPacked[txidStart : txidStart+32] + if int(req.ParentTxOffsets[txCount]) != totalParents { + return nil, errors.WrapGRPC(errors.NewInvalidArgumentError( + "parent_tx_offsets[txCount]=%d must equal total parent count %d", + req.ParentTxOffsets[txCount], totalParents)) + } - // Reconstruct TxInpoints from columnar data WITHOUT deserialization - // This is the key optimization - we build TxInpoints directly from pre-parsed data - parentHashStart := req.ParentTxOffsets[i] - parentHashEnd := req.ParentTxOffsets[i+1] - numParentHashes := parentHashEnd - parentHashStart + if int(req.VoutIdxsTxOffsets[txCount]) != voutIdxsLen { + return nil, errors.WrapGRPC(errors.NewInvalidArgumentError( + "vout_idxs_tx_offsets[txCount]=%d must equal len(vout_idxs_packed)=%d", + req.VoutIdxsTxOffsets[txCount], voutIdxsLen)) + } - // Pre-allocate slices with exact capacity to avoid reallocation - parentTxHashes := make([]chainhash.Hash, numParentHashes) - idxs := make([][]uint32, numParentHashes) + for i := 1; i <= txCount; i++ { + if req.ParentTxOffsets[i] < req.ParentTxOffsets[i-1] { + return nil, errors.WrapGRPC(errors.NewInvalidArgumentError( + "parent_tx_offsets must be monotonic non-decreasing at index %d (%d < %d)", + i, req.ParentTxOffsets[i], req.ParentTxOffsets[i-1])) + } - for j := uint32(0); j < numParentHashes; j++ { - parentHashIdx := parentHashStart + j + if req.VoutIdxsTxOffsets[i] < req.VoutIdxsTxOffsets[i-1] { + return nil, errors.WrapGRPC(errors.NewInvalidArgumentError( + "vout_idxs_tx_offsets must be monotonic non-decreasing at index %d (%d < %d)", + i, req.VoutIdxsTxOffsets[i], req.VoutIdxsTxOffsets[i-1])) + } + } - // Extract parent hash (32 bytes) - no allocation, direct copy - hashOffset := parentHashIdx * 32 - copy(parentTxHashes[j][:], req.ParentTxHashesPacked[hashOffset:hashOffset+32]) + if ba.settings.BlockAssembly.Disabled { + return &blockassembly_api.AddTxBatchResponse{Ok: true}, nil + } - // Extract vout indices for this parent hash - voutIdxStart := req.VoutIdxOffsets[parentHashIdx] - voutIdxEnd := req.VoutIdxOffsets[parentHashIdx+1] + // Build batch arrays — three allocations for the whole batch. + nodes := make([]subtreepkg.Node, txCount) + txInpointsArr := make([]subtreepkg.TxInpoints, txCount) + txInpointsList := make([]*subtreepkg.TxInpoints, txCount) - // Reference the vout indices slice directly - no allocation - idxs[j] = req.ParentVoutIndices[voutIdxStart:voutIdxEnd] - } + // Reinterpret the packed parent-hash byte buffer as []chainhash.Hash once + // for the whole batch. chainhash.Hash is [32]byte with byte alignment, so + // a []byte backing is byte-aligned and safe to reinterpret. Each per-tx + // slice of `parents` below is just a slice header — zero allocation. + // + // Aliasing assumption: proto.Unmarshal decodes `bytes` fields into a fresh + // Go-heap allocation (verified for google.golang.org/protobuf v1.36.x's + // consumeBytes path). If a future zero-copy codec ever lands that aliases + // the gRPC receive buffer into `ParentTxHashesPacked`, this aliasing must + // be reconsidered — the receive buffer is freed after handler return, + // which would invalidate any TxInpoints we hand off downstream. + var parents []chainhash.Hash + if totalParents > 0 { + parents = unsafe.Slice( + (*chainhash.Hash)(unsafe.Pointer(&req.ParentTxHashesPacked[0])), + totalParents, + ) + } + + storeTxInpoints := ba.settings.BlockAssembly.StoreTxInpointsForSubtreeMeta - // Build node and txInpoints for this transaction + for i := 0; i < txCount; i++ { + // Extract TXID (32 bytes) — no allocation, copy into the Node. + txidStart := i * 32 nodes[i] = subtreepkg.Node{ - Hash: chainhash.Hash(txid), + Hash: chainhash.Hash(req.TxidsPacked[txidStart : txidStart+32]), Fee: req.Fees[i], SizeInBytes: req.Sizes[i], } - if ba.settings.BlockAssembly.StoreTxInpointsForSubtreeMeta { - txInpointsList[i] = &subtreepkg.TxInpoints{ - ParentTxHashes: parentTxHashes, - Idxs: idxs, - } - } else { - txInpointsList[i] = &subtreepkg.TxInpoints{} + if storeTxInpoints { + // Two slice operations per tx: parent hashes and packed voutIdxs. + // Bounds are guaranteed by the offset-array validation above, so + // these slice expressions cannot panic. + parentSlice := parents[req.ParentTxOffsets[i]:req.ParentTxOffsets[i+1]] + voutSlice := req.VoutIdxsPacked[req.VoutIdxsTxOffsets[i]:req.VoutIdxsTxOffsets[i+1]] + txInpointsArr[i] = subtreepkg.NewTxInpointsFromPacked(parentSlice, voutSlice) } + // else: txInpointsArr[i] is the zero-value TxInpoints, which is the + // desired behaviour when not storing inpoints for subtree meta. + + txInpointsList[i] = &txInpointsArr[i] } prometheusBlockAssemblyAddTxCounter.Add(float64(len(nodes))) // gosec:nolint - // Add entire batch in one call - if !ba.settings.BlockAssembly.Disabled { - ba.blockAssembler.AddTxBatch(nodes, txInpointsList) - } + ba.blockAssembler.AddTxBatch(nodes, txInpointsList) return &blockassembly_api.AddTxBatchResponse{Ok: true}, nil } diff --git a/services/blockassembly/blockassembly_api/blockassembly_api.pb.go b/services/blockassembly/blockassembly_api/blockassembly_api.pb.go index b60132f0f0..5b1c9bc5b0 100644 --- a/services/blockassembly/blockassembly_api/blockassembly_api.pb.go +++ b/services/blockassembly/blockassembly_api/blockassembly_api.pb.go @@ -340,18 +340,32 @@ type AddTxBatchColumnarRequest struct { // - Transaction 1: 4 parent hashes (hashes 3-6) // - Transaction 2: 3 parent hashes (hashes 7-9) ParentTxOffsets []uint32 `protobuf:"varint,7,rep,packed,name=parent_tx_offsets,json=parentTxOffsets,proto3" json:"parent_tx_offsets,omitempty"` - // All parent vout indices for all transactions, flattened and concatenated. - // These are the Idxs from TxInpoints (the uint32 indices for each parent hash). - // Use vout_idx_offsets to identify boundaries. - ParentVoutIndices []uint32 `protobuf:"varint,8,rep,packed,name=parent_vout_indices,json=parentVoutIndices,proto3" json:"parent_vout_indices,omitempty"` - // Offsets into parent_vout_indices for each parent hash. - // Length must be exactly (total number of parent hashes) + 1. - // This creates a 2-level offset structure: - // 1. parent_tx_offsets identifies which parent hashes belong to each transaction - // 2. vout_idx_offsets identifies which vout indices belong to each parent hash - VoutIdxOffsets []uint32 `protobuf:"varint,9,rep,packed,name=vout_idx_offsets,json=voutIdxOffsets,proto3" json:"vout_idx_offsets,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // All parent vout indices for all transactions in the count-prefixed packed + // layout consumed directly by go-subtree.TxInpoints. For each parent (in + // parent_tx_hashes_packed order) one uint32 count word is followed by that + // many uint32 vout-value words, concatenated end-to-end across all txs. + // + // This is the exact shape TxInpoints.voutIdxs stores internally, so the + // Server aliases a per-tx slice of this buffer straight into a TxInpoints + // with zero copy (NewTxInpointsFromPacked). The buffer's lifetime is bound + // to the request; gRPC keeps it alive until every TxInpoints derived from + // it is no longer referenced. + // + // Assigned a fresh field number (10) rather than reusing the old field 8 + // because the old field carried different semantics and the wire types + // match — silent misinterpretation under version skew would otherwise be + // possible. + VoutIdxsPacked []uint32 `protobuf:"varint,10,rep,packed,name=vout_idxs_packed,json=voutIdxsPacked,proto3" json:"vout_idxs_packed,omitempty"` + // Offsets into vout_idxs_packed for each transaction. + // Length must be exactly transaction_count + 1. + // Each offset is an index into vout_idxs_packed (in uint32 units). + // Example: [0, 5, 8] represents 2 transactions, the first occupying + // vout_idxs_packed[0:5] and the second occupying vout_idxs_packed[5:8]. + // + // Assigned a fresh field number (11) — see note on field 10. + VoutIdxsTxOffsets []uint32 `protobuf:"varint,11,rep,packed,name=vout_idxs_tx_offsets,json=voutIdxsTxOffsets,proto3" json:"vout_idxs_tx_offsets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *AddTxBatchColumnarRequest) Reset() { @@ -419,16 +433,16 @@ func (x *AddTxBatchColumnarRequest) GetParentTxOffsets() []uint32 { return nil } -func (x *AddTxBatchColumnarRequest) GetParentVoutIndices() []uint32 { +func (x *AddTxBatchColumnarRequest) GetVoutIdxsPacked() []uint32 { if x != nil { - return x.ParentVoutIndices + return x.VoutIdxsPacked } return nil } -func (x *AddTxBatchColumnarRequest) GetVoutIdxOffsets() []uint32 { +func (x *AddTxBatchColumnarRequest) GetVoutIdxsTxOffsets() []uint32 { if x != nil { - return x.VoutIdxOffsets + return x.VoutIdxsTxOffsets } return nil } @@ -1204,15 +1218,17 @@ const file_services_blockassembly_blockassembly_api_blockassembly_api_proto_rawD "\x11AddTxBatchRequest\x12?\n" + "\n" + "txRequests\x18\x01 \x03(\v2\x1f.blockassembly_api.AddTxRequestR\n" + - "txRequests\"\xa5\x02\n" + + "txRequests\"\xd9\x02\n" + "\x19AddTxBatchColumnarRequest\x12!\n" + "\ftxids_packed\x18\x01 \x01(\fR\vtxidsPacked\x12\x12\n" + "\x04fees\x18\x02 \x03(\x04R\x04fees\x12\x14\n" + "\x05sizes\x18\x03 \x03(\x04R\x05sizes\x125\n" + "\x17parent_tx_hashes_packed\x18\x06 \x01(\fR\x14parentTxHashesPacked\x12*\n" + - "\x11parent_tx_offsets\x18\a \x03(\rR\x0fparentTxOffsets\x12.\n" + - "\x13parent_vout_indices\x18\b \x03(\rR\x11parentVoutIndices\x12(\n" + - "\x10vout_idx_offsets\x18\t \x03(\rR\x0evoutIdxOffsets\"E\n" + + "\x11parent_tx_offsets\x18\a \x03(\rR\x0fparentTxOffsets\x12(\n" + + "\x10vout_idxs_packed\x18\n" + + " \x03(\rR\x0evoutIdxsPacked\x12/\n" + + "\x14vout_idxs_tx_offsets\x18\v \x03(\rR\x11voutIdxsTxOffsetsJ\x04\b\b\x10\tJ\x04\b\t\x10\n" + + "R\x13parent_vout_indicesR\x10vout_idx_offsets\"E\n" + "\x19GetMiningCandidateRequest\x12(\n" + "\x0fincludeSubtrees\x18\x01 \x01(\bR\x0fincludeSubtrees\"%\n" + "\x0fRemoveTxRequest\x12\x12\n" + diff --git a/services/blockassembly/blockassembly_api/blockassembly_api.proto b/services/blockassembly/blockassembly_api/blockassembly_api.proto index eb6ef3d7d6..e36f49d3dc 100644 --- a/services/blockassembly/blockassembly_api/blockassembly_api.proto +++ b/services/blockassembly/blockassembly_api/blockassembly_api.proto @@ -143,6 +143,14 @@ message AddTxBatchRequest { // and sent in columnar format. This shifts deserialization work away from the single-machine Server. // The Server reconstructs TxInpoints directly from columnar data WITHOUT deserialization overhead. message AddTxBatchColumnarRequest { + // Fields 8 and 9 previously held parent_vout_indices + vout_idx_offsets + // (per-parent) with different semantics from the current packed layout. + // They are reserved to prevent silent misinterpretation under version skew: + // an old client sending field 8/9 to a new server would otherwise decode + // as the new fields with corrupt content. The new fields are 10/11 below. + reserved 8, 9; + reserved "parent_vout_indices", "vout_idx_offsets"; + // All transaction IDs concatenated (each exactly 32 bytes). // Total length must be divisible by 32. // Transaction count = len(txids_packed) / 32 @@ -170,17 +178,31 @@ message AddTxBatchColumnarRequest { // - Transaction 2: 3 parent hashes (hashes 7-9) repeated uint32 parent_tx_offsets = 7; - // All parent vout indices for all transactions, flattened and concatenated. - // These are the Idxs from TxInpoints (the uint32 indices for each parent hash). - // Use vout_idx_offsets to identify boundaries. - repeated uint32 parent_vout_indices = 8; - - // Offsets into parent_vout_indices for each parent hash. - // Length must be exactly (total number of parent hashes) + 1. - // This creates a 2-level offset structure: - // 1. parent_tx_offsets identifies which parent hashes belong to each transaction - // 2. vout_idx_offsets identifies which vout indices belong to each parent hash - repeated uint32 vout_idx_offsets = 9; + // All parent vout indices for all transactions in the count-prefixed packed + // layout consumed directly by go-subtree.TxInpoints. For each parent (in + // parent_tx_hashes_packed order) one uint32 count word is followed by that + // many uint32 vout-value words, concatenated end-to-end across all txs. + // + // This is the exact shape TxInpoints.voutIdxs stores internally, so the + // Server aliases a per-tx slice of this buffer straight into a TxInpoints + // with zero copy (NewTxInpointsFromPacked). The buffer's lifetime is bound + // to the request; gRPC keeps it alive until every TxInpoints derived from + // it is no longer referenced. + // + // Assigned a fresh field number (10) rather than reusing the old field 8 + // because the old field carried different semantics and the wire types + // match — silent misinterpretation under version skew would otherwise be + // possible. + repeated uint32 vout_idxs_packed = 10; + + // Offsets into vout_idxs_packed for each transaction. + // Length must be exactly transaction_count + 1. + // Each offset is an index into vout_idxs_packed (in uint32 units). + // Example: [0, 5, 8] represents 2 transactions, the first occupying + // vout_idxs_packed[0:5] and the second occupying vout_idxs_packed[5:8]. + // + // Assigned a fresh field number (11) — see note on field 10. + repeated uint32 vout_idxs_tx_offsets = 11; } // Request for retrieving a mining candidate block template. diff --git a/services/blockassembly/data_test.go b/services/blockassembly/data_test.go index 6220e667a3..b0e6741623 100644 --- a/services/blockassembly/data_test.go +++ b/services/blockassembly/data_test.go @@ -3,6 +3,7 @@ package blockassembly import ( "testing" + "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/go-subtree" "github.com/stretchr/testify/assert" @@ -28,20 +29,23 @@ func TestData_Bytes(t *testing.T) { }) t.Run("should return the correct bytes, with parents", func(t *testing.T) { + mkIn := func(parent *chainhash.Hash, vout uint32) *bt.Input { + in := &bt.Input{PreviousTxOutIndex: vout} + require.NoError(t, in.PreviousTxIDAdd(parent)) + return in + } + + ti, err := subtree.NewTxInpointsFromInputs([]*bt.Input{ + mkIn(hash1, 1), mkIn(hash1, 2), + mkIn(hash2, 3), mkIn(hash2, 4), + }) + require.NoError(t, err) + d := &Data{ TxIDChainHash: *hash0, Fee: 1, Size: 2, - TxInpoints: subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{ - *hash1, - *hash2, - }, - Idxs: [][]uint32{ - {1, 2}, - {3, 4}, - }, - }, + TxInpoints: ti, } b := d.Bytes() diff --git a/services/blockassembly/filter_transactions_test.go b/services/blockassembly/filter_transactions_test.go index 02fa47a8e1..4fb36bfd74 100644 --- a/services/blockassembly/filter_transactions_test.go +++ b/services/blockassembly/filter_transactions_test.go @@ -64,11 +64,9 @@ func TestValidateParentChain_BatchingAndOrdering(t *testing.T) { Fee: 1000, SizeInBytes: 250, }, - TxInpoints: &subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{{}}, // Empty hash means mined parent - Idxs: [][]uint32{{0}}, - }, - CreatedAt: i, + // Empty hash means mined parent + TxInpoints: singleParentInpointsPtr(chainhash.Hash{}, 0), + CreatedAt: i, } unminedTxs = append(unminedTxs, tx) } @@ -88,11 +86,8 @@ func TestValidateParentChain_BatchingAndOrdering(t *testing.T) { Fee: 1000, SizeInBytes: 250, }, - TxInpoints: &subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{parentTxHashes[i]}, - Idxs: [][]uint32{{0}}, - }, - CreatedAt: 50 + i, + TxInpoints: singleParentInpointsPtr(parentTxHashes[i], 0), + CreatedAt: 50 + i, } unminedTxs = append(unminedTxs, tx) } @@ -109,11 +104,9 @@ func TestValidateParentChain_BatchingAndOrdering(t *testing.T) { Fee: 1000, SizeInBytes: 250, }, - TxInpoints: &subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{childTxHashes[0]}, // Depends on tx at index 50 - Idxs: [][]uint32{{0}}, - }, - CreatedAt: 100, + // Depends on tx at index 50 + TxInpoints: singleParentInpointsPtr(childTxHashes[0], 0), + CreatedAt: 100, } unminedTxs = append(unminedTxs, grandchildTx) @@ -224,11 +217,8 @@ func TestValidateParentChain_BatchingAndOrdering(t *testing.T) { Fee: 1000, SizeInBytes: 250, }, - TxInpoints: &subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{parentHash}, - Idxs: [][]uint32{{0}}, - }, - CreatedAt: 0, + TxInpoints: singleParentInpointsPtr(parentHash, 0), + CreatedAt: 0, } // Parent transaction (index 1) - no unmined parents @@ -238,11 +228,9 @@ func TestValidateParentChain_BatchingAndOrdering(t *testing.T) { Fee: 1000, SizeInBytes: 250, }, - TxInpoints: &subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{{}}, // Empty hash = mined parent - Idxs: [][]uint32{{0}}, - }, - CreatedAt: 1, + // Empty hash = mined parent + TxInpoints: singleParentInpointsPtr(chainhash.Hash{}, 0), + CreatedAt: 1, } unminedTxs := []*utxo.UnminedTransaction{childTx, parentTx} @@ -336,11 +324,8 @@ func TestValidateParentChain_RejectsChildOfConflictingParent(t *testing.T) { Fee: 1000, SizeInBytes: 250, }, - TxInpoints: &subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{parentHash}, - Idxs: [][]uint32{{0}}, - }, - CreatedAt: 1, + TxInpoints: singleParentInpointsPtr(parentHash, 0), + CreatedAt: 1, } unminedTxs := []*utxo.UnminedTransaction{childTx} @@ -408,10 +393,7 @@ func TestValidateParentChain_BatchDecorateRequestsConflicting(t *testing.T) { Fee: 1000, SizeInBytes: 250, }, - TxInpoints: &subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{parentHash}, - Idxs: [][]uint32{{0}}, - }, + TxInpoints: singleParentInpointsPtr(parentHash, 0), } var capturedFields []fields.FieldName @@ -486,17 +468,17 @@ func TestValidateParentChain_RecursivelyFiltersConflictingDescendants(t *testing childB := &utxo.UnminedTransaction{ Node: &subtree.Node{Hash: bHash, Fee: 1000, SizeInBytes: 250}, - TxInpoints: &subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{conflictingParentHash}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpointsPtr(conflictingParentHash, 0), CreatedAt: 1, } childC := &utxo.UnminedTransaction{ Node: &subtree.Node{Hash: cHash, Fee: 1000, SizeInBytes: 250}, - TxInpoints: &subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{bHash}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpointsPtr(bHash, 0), CreatedAt: 2, } childD := &utxo.UnminedTransaction{ Node: &subtree.Node{Hash: dHash, Fee: 1000, SizeInBytes: 250}, - TxInpoints: &subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{cHash}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpointsPtr(cHash, 0), CreatedAt: 3, } @@ -591,12 +573,12 @@ func TestValidateParentChain_RecursivelyFiltersOtherInvalidDescendants(t *testin childB := &utxo.UnminedTransaction{ Node: &subtree.Node{Hash: bHash, Fee: 1000, SizeInBytes: 250}, - TxInpoints: &subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{missingParentHash}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpointsPtr(missingParentHash, 0), CreatedAt: 1, } childC := &utxo.UnminedTransaction{ Node: &subtree.Node{Hash: cHash, Fee: 1000, SizeInBytes: 250}, - TxInpoints: &subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{bHash}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpointsPtr(bHash, 0), CreatedAt: 2, } diff --git a/services/blockassembly/inpoints_helper_test.go b/services/blockassembly/inpoints_helper_test.go new file mode 100644 index 0000000000..ace594846f --- /dev/null +++ b/services/blockassembly/inpoints_helper_test.go @@ -0,0 +1,27 @@ +package blockassembly + +import ( + "github.com/bsv-blockchain/go-bt/v2" + "github.com/bsv-blockchain/go-bt/v2/chainhash" + "github.com/bsv-blockchain/go-subtree" +) + +// singleParentInpointsPtr builds a *TxInpoints with one parent and one vout, +// matching the pre-packed-layout pattern +// `&subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{X}, Idxs: [][]uint32{{vout}}}`. +// +// Tests use this to replace direct struct-literal construction now that the +// Idxs field has been removed. +func singleParentInpointsPtr(parent chainhash.Hash, vout uint32) *subtree.TxInpoints { + in := &bt.Input{PreviousTxOutIndex: vout} + if err := in.PreviousTxIDAdd(&parent); err != nil { + panic(err) + } + + ti, err := subtree.NewTxInpointsFromInputs([]*bt.Input{in}) + if err != nil { + panic(err) + } + + return &ti +} diff --git a/services/blockassembly/parent_validation_ordering_test.go b/services/blockassembly/parent_validation_ordering_test.go index 72979f0da3..134cf8aa4c 100644 --- a/services/blockassembly/parent_validation_ordering_test.go +++ b/services/blockassembly/parent_validation_ordering_test.go @@ -4,6 +4,7 @@ package blockassembly import ( "testing" + "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/go-subtree" "github.com/bsv-blockchain/teranode/stores/utxo" @@ -16,29 +17,28 @@ func createTestTx(txID string, parentIDs ...string) *utxo.UnminedTransaction { var txInpoints subtree.TxInpoints if len(parentIDs) > 0 { - // Create parent transaction hashes and indices - parentHashes := make([]chainhash.Hash, 0, len(parentIDs)) - idxs := make([][]uint32, 0, len(parentIDs)) + // For simplicity, always use output index 0. Build one bt.Input per + // parentID and let NewTxInpointsFromInputs handle the dedup of + // repeated parent hashes — this matches the prior parentMap step. + inputs := make([]*bt.Input, 0, len(parentIDs)) - // Group parents by hash - parentMap := make(map[string][]uint32) for _, parentID := range parentIDs { - if _, exists := parentMap[parentID]; !exists { - parentMap[parentID] = []uint32{} + parentHash, _ := chainhash.NewHashFromStr(parentID) + in := &bt.Input{PreviousTxOutIndex: 0} + + if err := in.PreviousTxIDAdd(parentHash); err != nil { + panic(err) } - // For simplicity, always use output index 0 - parentMap[parentID] = append(parentMap[parentID], 0) - } - // Build the arrays - for parentID, indices := range parentMap { - parentHash, _ := chainhash.NewHashFromStr(parentID) - parentHashes = append(parentHashes, *parentHash) - idxs = append(idxs, indices) + inputs = append(inputs, in) } - txInpoints.ParentTxHashes = parentHashes - txInpoints.Idxs = idxs + var err error + + txInpoints, err = subtree.NewTxInpointsFromInputs(inputs) + if err != nil { + panic(err) + } } // Create hash from the txID string diff --git a/services/blockassembly/server_columnar_bench_test.go b/services/blockassembly/server_columnar_bench_test.go new file mode 100644 index 0000000000..46c7249886 --- /dev/null +++ b/services/blockassembly/server_columnar_bench_test.go @@ -0,0 +1,112 @@ +package blockassembly + +import ( + "context" + "testing" + + "github.com/bsv-blockchain/go-bt/v2/chainhash" + "github.com/bsv-blockchain/teranode/services/blockassembly/blockassembly_api" +) + +// BenchmarkAddTxBatchColumnar_Validation measures the overhead of the offset +// validation block before the per-tx loop. Single-parent / single-vout txs, +// batch of 1000 — representative of the steady-state workload. +// +// To isolate the validation cost, the bench builds a well-formed batch once +// and re-submits it. With BlockAssembly.Disabled=true the handler returns +// immediately after validation, so the per-iteration cost is dominated by +// proto unmarshal (constant across all variants) plus our validation. +func BenchmarkAddTxBatchColumnar_Validation(b *testing.B) { + ba, _ := setupServer(&testing.T{}) + ba.settings.BlockAssembly.Disabled = true + ba.settings.BlockAssembly.StoreTxInpointsForSubtreeMeta = true + + const batchSize = 1000 + + txidsPacked := make([]byte, batchSize*32) + parentTxHashesPacked := make([]byte, batchSize*32) // 1 parent per tx + fees := make([]uint64, batchSize) + sizes := make([]uint64, batchSize) + parentTxOffsets := make([]uint32, batchSize+1) + voutIdxsPacked := make([]uint32, batchSize*2) // [count=1, vout=0] per tx + voutIdxsTxOffsets := make([]uint32, batchSize+1) + + for i := 0; i < batchSize; i++ { + txid := chainhash.Hash{byte(i), byte(i >> 8)} + copy(txidsPacked[i*32:(i+1)*32], txid[:]) + + parent := chainhash.Hash{byte(i + 1), byte(i + 1>>8)} + copy(parentTxHashesPacked[i*32:(i+1)*32], parent[:]) + + fees[i] = 1000 + sizes[i] = 250 + parentTxOffsets[i+1] = uint32(i + 1) + voutIdxsPacked[i*2] = 1 // count + voutIdxsPacked[i*2+1] = 0 // vout value + voutIdxsTxOffsets[i+1] = uint32((i + 1) * 2) + } + + req := &blockassembly_api.AddTxBatchColumnarRequest{ + TxidsPacked: txidsPacked, + Fees: fees, + Sizes: sizes, + ParentTxHashesPacked: parentTxHashesPacked, + ParentTxOffsets: parentTxOffsets, + VoutIdxsPacked: voutIdxsPacked, + VoutIdxsTxOffsets: voutIdxsTxOffsets, + } + + ctx := context.Background() + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + if _, err := ba.AddTxBatchColumnar(ctx, req); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkOffsetValidationLoop isolates just the monotonicity + endpoint +// check cost over a 1000-tx batch's offset arrays. This is the work added +// by the security hardening — nothing else. +func BenchmarkOffsetValidationLoop(b *testing.B) { + const txCount = 1000 + + parentTxOffsets := make([]uint32, txCount+1) + voutIdxsTxOffsets := make([]uint32, txCount+1) + for i := 0; i <= txCount; i++ { + parentTxOffsets[i] = uint32(i) + voutIdxsTxOffsets[i] = uint32(i * 2) + } + + totalParents := txCount + voutIdxsLen := txCount * 2 + + b.ResetTimer() + b.ReportAllocs() + + for n := 0; n < b.N; n++ { + // Mirrors the validation block in AddTxBatchColumnar verbatim, so + // the cost reported here is exactly what the production code pays + // per batch. + _ = parentTxOffsets[0] + _ = voutIdxsTxOffsets[0] + _ = int(parentTxOffsets[txCount]) != totalParents + _ = int(voutIdxsTxOffsets[txCount]) != voutIdxsLen + + var bad bool + for i := 1; i <= txCount; i++ { + if parentTxOffsets[i] < parentTxOffsets[i-1] { + bad = true + break + } + if voutIdxsTxOffsets[i] < voutIdxsTxOffsets[i-1] { + bad = true + break + } + } + _ = bad + } +} diff --git a/services/blockassembly/server_columnar_test.go b/services/blockassembly/server_columnar_test.go index e408221768..5a67394079 100644 --- a/services/blockassembly/server_columnar_test.go +++ b/services/blockassembly/server_columnar_test.go @@ -4,12 +4,37 @@ import ( "context" "testing" + "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" subtreepkg "github.com/bsv-blockchain/go-subtree" "github.com/bsv-blockchain/teranode/services/blockassembly/blockassembly_api" "github.com/stretchr/testify/require" ) +// inpointsForParents builds wire-format TxInpoints bytes for tests with +// one or more parents and N vouts per parent (matching old patterns). +func inpointsForParents(t *testing.T, parents []chainhash.Hash, vouts [][]uint32) []byte { + t.Helper() + + inputs := make([]*bt.Input, 0) + + for i := range parents { + for _, v := range vouts[i] { + in := &bt.Input{PreviousTxOutIndex: v} + require.NoError(t, in.PreviousTxIDAdd(&parents[i])) + inputs = append(inputs, in) + } + } + + ti, err := subtreepkg.NewTxInpointsFromInputs(inputs) + require.NoError(t, err) + + b, err := ti.Serialize() + require.NoError(t, err) + + return b +} + // TestAddTxBatchColumnar_Success verifies that the columnar batch format processes transactions correctly. func TestAddTxBatchColumnar_Success(t *testing.T) { ba, _ := setupServer(t) @@ -21,13 +46,12 @@ func TestAddTxBatchColumnar_Success(t *testing.T) { sizes := make([]uint64, txCount) parentTxOffsets := make([]uint32, txCount+1) parentTxHashesPacked := make([]byte, 0) - parentVoutIndices := make([]uint32, 0) - voutIdxOffsets := make([]uint32, 1) // Start with 0 + voutIdxsPacked := make([]uint32, 0) + voutIdxsTxOffsets := make([]uint32, txCount+1) currentParentHashCount := uint32(0) - currentVoutIdxCount := uint32(0) parentTxOffsets[0] = 0 - voutIdxOffsets[0] = 0 + voutIdxsTxOffsets[0] = 0 // Generate test data for i := 0; i < txCount; i++ { @@ -40,7 +64,8 @@ func TestAddTxBatchColumnar_Success(t *testing.T) { fees[i] = uint64(1000 * (i + 1)) sizes[i] = uint64(250 + i*10) - // Create TxInpoints with i+1 inputs + // Create TxInpoints with i+1 inputs, one vout each, count-prefixed + // packed layout (matches TxInpoints.voutIdxs). numParentHashes := i + 1 for j := 0; j < numParentHashes; j++ { prevTxid := chainhash.Hash{} @@ -48,12 +73,12 @@ func TestAddTxBatchColumnar_Success(t *testing.T) { parentTxHashesPacked = append(parentTxHashesPacked, prevTxid[:]...) currentParentHashCount++ - // Each parent hash has one vout index - parentVoutIndices = append(parentVoutIndices, uint32(j)) - currentVoutIdxCount++ - voutIdxOffsets = append(voutIdxOffsets, currentVoutIdxCount) + // One vout per parent: [count=1, vout=j] + voutIdxsPacked = append(voutIdxsPacked, 1, uint32(j)) } + parentTxOffsets[i+1] = currentParentHashCount + voutIdxsTxOffsets[i+1] = uint32(len(voutIdxsPacked)) } // Create columnar request @@ -63,8 +88,8 @@ func TestAddTxBatchColumnar_Success(t *testing.T) { Sizes: sizes, ParentTxHashesPacked: parentTxHashesPacked, ParentTxOffsets: parentTxOffsets, - ParentVoutIndices: parentVoutIndices, - VoutIdxOffsets: voutIdxOffsets, + VoutIdxsPacked: voutIdxsPacked, + VoutIdxsTxOffsets: voutIdxsTxOffsets, } // Call AddTxBatchColumnar @@ -87,8 +112,8 @@ func TestAddTxBatchColumnar_ValidatesTxidsLength(t *testing.T) { Sizes: []uint64{250}, ParentTxHashesPacked: []byte{}, ParentTxOffsets: []uint32{0, 0}, - ParentVoutIndices: []uint32{}, - VoutIdxOffsets: []uint32{0}, + VoutIdxsPacked: []uint32{}, + VoutIdxsTxOffsets: []uint32{0, 0}, } _, err := ba.AddTxBatchColumnar(context.Background(), req) @@ -106,8 +131,8 @@ func TestAddTxBatchColumnar_ValidatesEmptyBatch(t *testing.T) { Sizes: []uint64{}, ParentTxHashesPacked: []byte{}, ParentTxOffsets: []uint32{0}, - ParentVoutIndices: []uint32{}, - VoutIdxOffsets: []uint32{0}, + VoutIdxsPacked: []uint32{}, + VoutIdxsTxOffsets: []uint32{0}, } _, err := ba.AddTxBatchColumnar(context.Background(), req) @@ -126,8 +151,8 @@ func TestAddTxBatchColumnar_ValidatesArrayLengths(t *testing.T) { Sizes: []uint64{250}, ParentTxHashesPacked: []byte{}, ParentTxOffsets: []uint32{0, 0}, - ParentVoutIndices: []uint32{}, - VoutIdxOffsets: []uint32{0}, + VoutIdxsPacked: []uint32{}, + VoutIdxsTxOffsets: []uint32{0, 0}, } _, err := ba.AddTxBatchColumnar(context.Background(), req) @@ -146,8 +171,8 @@ func TestAddTxBatchColumnar_ValidatesParentTxOffsets(t *testing.T) { Sizes: []uint64{250}, ParentTxHashesPacked: []byte{}, ParentTxOffsets: []uint32{0}, // Should be 2 elements (txCount+1), not 1 - ParentVoutIndices: []uint32{}, - VoutIdxOffsets: []uint32{0}, + VoutIdxsPacked: []uint32{}, + VoutIdxsTxOffsets: []uint32{0, 0}, } _, err := ba.AddTxBatchColumnar(context.Background(), req) @@ -166,8 +191,8 @@ func TestAddTxBatchColumnar_ValidatesParentHashesLength(t *testing.T) { Sizes: []uint64{250}, ParentTxHashesPacked: make([]byte, 33), // Not divisible by 32 ParentTxOffsets: []uint32{0, 1}, - ParentVoutIndices: []uint32{0}, - VoutIdxOffsets: []uint32{0, 1}, + VoutIdxsPacked: []uint32{1, 0}, // [count=1, vout=0] + VoutIdxsTxOffsets: []uint32{0, 2}, } _, err := ba.AddTxBatchColumnar(context.Background(), req) @@ -175,8 +200,9 @@ func TestAddTxBatchColumnar_ValidatesParentHashesLength(t *testing.T) { require.Contains(t, err.Error(), "parent_tx_hashes_packed length must be divisible by 32") } -// TestAddTxBatchColumnar_ValidatesVoutIdxOffsets checks vout idx offset validation. -func TestAddTxBatchColumnar_ValidatesVoutIdxOffsets(t *testing.T) { +// TestAddTxBatchColumnar_ValidatesVoutIdxsTxOffsets checks the per-tx vout +// offset array validation. +func TestAddTxBatchColumnar_ValidatesVoutIdxsTxOffsets(t *testing.T) { ba, _ := setupServer(t) txid := chainhash.Hash{} @@ -187,13 +213,105 @@ func TestAddTxBatchColumnar_ValidatesVoutIdxOffsets(t *testing.T) { Sizes: []uint64{250}, ParentTxHashesPacked: parentHash[:], // 1 parent hash ParentTxOffsets: []uint32{0, 1}, - ParentVoutIndices: []uint32{0}, - VoutIdxOffsets: []uint32{0}, // Should be 2 elements (totalParentHashes+1), not 1 + VoutIdxsPacked: []uint32{1, 0}, + VoutIdxsTxOffsets: []uint32{0}, // Should be 2 elements (txCount+1), not 1 } _, err := ba.AddTxBatchColumnar(context.Background(), req) require.Error(t, err) - require.Contains(t, err.Error(), "vout_idx_offsets must have exactly (total_parent_hashes+1) elements") + require.Contains(t, err.Error(), "vout_idxs_tx_offsets must have exactly txCount+1 elements") +} + +// TestAddTxBatchColumnar_RejectsOOBParentTxOffsets verifies the handler +// rejects a malformed ParentTxOffsets pointing past the parent hash buffer +// rather than panicking inside the per-tx loop. Regression test for the +// security audit (see PR #889): grpc-go does not recover handler panics, +// so a single bad request would otherwise crash block-assembly. +func TestAddTxBatchColumnar_RejectsOOBParentTxOffsets(t *testing.T) { + ba, _ := setupServer(t) + + enableStoreTxInpoints(t, ba) + + txid := chainhash.Hash{} + parentHash := chainhash.Hash{} + req := &blockassembly_api.AddTxBatchColumnarRequest{ + TxidsPacked: txid[:], + Fees: []uint64{1000}, + Sizes: []uint64{250}, + ParentTxHashesPacked: parentHash[:], // 1 parent hash → totalParents=1 + ParentTxOffsets: []uint32{0, 99}, // claims 99 parents — OOB + VoutIdxsPacked: []uint32{1, 0}, + VoutIdxsTxOffsets: []uint32{0, 2}, + } + + require.NotPanics(t, func() { + _, err := ba.AddTxBatchColumnar(context.Background(), req) + require.Error(t, err) + require.Contains(t, err.Error(), "parent_tx_offsets[txCount]") + }) +} + +// TestAddTxBatchColumnar_RejectsNonMonotonicOffsets ensures non-monotonic +// offset arrays produce a clean error rather than a slice-bounds panic. +func TestAddTxBatchColumnar_RejectsNonMonotonicOffsets(t *testing.T) { + ba, _ := setupServer(t) + + enableStoreTxInpoints(t, ba) + + // Two parents but offsets dip in the middle. Endpoints still satisfy the + // totalParents/voutIdxsLen checks, so the monotonicity loop is what + // must catch this. + parents := make([]byte, 64) // 2 parent hashes → totalParents=2 + txids := make([]byte, 64) // 2 txs + req := &blockassembly_api.AddTxBatchColumnarRequest{ + TxidsPacked: txids, + Fees: []uint64{1000, 2000}, + Sizes: []uint64{250, 260}, + ParentTxHashesPacked: parents, + ParentTxOffsets: []uint32{0, 3, 2}, // 3 → 2: monotonicity violation + VoutIdxsPacked: []uint32{1, 0, 1, 1}, + VoutIdxsTxOffsets: []uint32{0, 2, 4}, + } + + require.NotPanics(t, func() { + _, err := ba.AddTxBatchColumnar(context.Background(), req) + require.Error(t, err) + require.Contains(t, err.Error(), "parent_tx_offsets must be monotonic") + }) +} + +// TestAddTxBatchColumnar_RejectsOOBVoutIdxsTxOffsets covers the vout side of +// the same exploit class. +func TestAddTxBatchColumnar_RejectsOOBVoutIdxsTxOffsets(t *testing.T) { + ba, _ := setupServer(t) + + enableStoreTxInpoints(t, ba) + + txid := chainhash.Hash{} + parentHash := chainhash.Hash{} + req := &blockassembly_api.AddTxBatchColumnarRequest{ + TxidsPacked: txid[:], + Fees: []uint64{1000}, + Sizes: []uint64{250}, + ParentTxHashesPacked: parentHash[:], + ParentTxOffsets: []uint32{0, 1}, + VoutIdxsPacked: []uint32{1, 0}, + VoutIdxsTxOffsets: []uint32{0, 999}, // OOB + } + + require.NotPanics(t, func() { + _, err := ba.AddTxBatchColumnar(context.Background(), req) + require.Error(t, err) + require.Contains(t, err.Error(), "vout_idxs_tx_offsets[txCount]") + }) +} + +// enableStoreTxInpoints flips the StoreTxInpointsForSubtreeMeta setting so +// the per-tx loop actually exercises the unsafe.Slice + offset arithmetic +// that the regression tests target. +func enableStoreTxInpoints(t *testing.T, ba *BlockAssembly) { + t.Helper() + ba.settings.BlockAssembly.StoreTxInpointsForSubtreeMeta = true } // TestConvertToColumnarFormat_Success verifies columnar conversion. @@ -211,28 +329,22 @@ func TestConvertToColumnarFormat_Success(t *testing.T) { txid := chainhash.Hash{} txid[0] = byte(i) - // Create TxInpoints - parentHashes := make([]chainhash.Hash, i+1) - idxs := make([][]uint32, i+1) + // Build i+1 parents, one vout each, into wire-format TxInpoints bytes. + parents := make([]chainhash.Hash, i+1) + vouts := make([][]uint32, i+1) for j := 0; j < i+1; j++ { prevTxid := chainhash.Hash{} prevTxid[0] = byte(j) - parentHashes[j] = prevTxid - idxs[j] = []uint32{uint32(j)} - } - inpoints := subtreepkg.TxInpoints{ - ParentTxHashes: parentHashes, - Idxs: idxs, + parents[j] = prevTxid + vouts[j] = []uint32{uint32(j)} } - inpointsBytes, err := inpoints.Serialize() - require.NoError(t, err) batch[i] = &batchItem{ req: &blockassembly_api.AddTxRequest{ Txid: txid[:], Fee: uint64(1000 * (i + 1)), Size: uint64(250 + i*10), - TxInpoints: inpointsBytes, + TxInpoints: inpointsForParents(t, parents, vouts), }, done: make(chan error, 1), } @@ -247,7 +359,8 @@ func TestConvertToColumnarFormat_Success(t *testing.T) { require.Equal(t, 3*32, len(columnar.TxidsPacked)) // 3 transactions × 32 bytes require.Equal(t, 3, len(columnar.Fees)) require.Equal(t, 3, len(columnar.Sizes)) - require.Equal(t, 4, len(columnar.ParentTxOffsets)) // txCount + 1 + require.Equal(t, 4, len(columnar.ParentTxOffsets)) // txCount + 1 + require.Equal(t, 4, len(columnar.VoutIdxsTxOffsets)) // txCount + 1 // Verify TXIDs are packed correctly for i := 0; i < 3; i++ { @@ -274,8 +387,10 @@ func TestConvertToColumnarFormat_Success(t *testing.T) { totalParentHashes := columnar.ParentTxOffsets[len(columnar.ParentTxOffsets)-1] require.Equal(t, int(totalParentHashes), len(columnar.ParentTxHashesPacked)/32) - // Verify vout idx offsets length - require.Equal(t, int(totalParentHashes)+1, len(columnar.VoutIdxOffsets)) + // Verify VoutIdxsPacked has the count-prefix layout. With i+1 parents + // (1 vout each) for tx i, each tx contributes (count=1 + value) = 2 + // entries per parent, so total = 2 * (1+2+3) = 12 entries. + require.Equal(t, 12, len(columnar.VoutIdxsPacked)) } // TestConvertToColumnarFormat_InvalidTxidLength verifies TXID length validation in conversion. @@ -329,35 +444,27 @@ func TestAddTxBatchColumnar_RoundTrip(t *testing.T) { settings: ba.settings, } - // Create batch with complex TxInpoints + // Create batch with complex TxInpoints — two parents, three vouts each. batch := make([]*batchItem, 2) for i := 0; i < 2; i++ { txid := chainhash.Hash{} txid[0] = byte(i + 10) - // Create TxInpoints with multiple vouts per parent - parentHashes := make([]chainhash.Hash, 2) - idxs := make([][]uint32, 2) + parents := make([]chainhash.Hash, 2) + vouts := make([][]uint32, 2) for j := 0; j < 2; j++ { prevTxid := chainhash.Hash{} prevTxid[0] = byte(j + 20) - parentHashes[j] = prevTxid - // Multiple vout indices for each parent - idxs[j] = []uint32{uint32(j), uint32(j + 10), uint32(j + 20)} - } - inpoints := subtreepkg.TxInpoints{ - ParentTxHashes: parentHashes, - Idxs: idxs, + parents[j] = prevTxid + vouts[j] = []uint32{uint32(j), uint32(j + 10), uint32(j + 20)} } - inpointsBytes, err := inpoints.Serialize() - require.NoError(t, err) batch[i] = &batchItem{ req: &blockassembly_api.AddTxRequest{ Txid: txid[:], Fee: uint64(5000 * (i + 1)), Size: uint64(500 + i*50), - TxInpoints: inpointsBytes, + TxInpoints: inpointsForParents(t, parents, vouts), }, done: make(chan error, 1), } diff --git a/services/blockassembly/server_test.go b/services/blockassembly/server_test.go index c28d12e499..62de94674c 100644 --- a/services/blockassembly/server_test.go +++ b/services/blockassembly/server_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" subtreepkg "github.com/bsv-blockchain/go-subtree" txmap "github.com/bsv-blockchain/go-tx-map" @@ -240,14 +241,12 @@ func TestGetBlockAssemblyBlockCandidate(t *testing.T) { // Use a common parent hash for all transactions (simulating they all spend from same output) genesisHash := chainhash.HashH([]byte("genesis")) for i := uint64(0); i < 10; i++ { + // Different output index for each tx server.blockAssembler.AddTxBatch([]subtreepkg.Node{{ Hash: chainhash.HashH([]byte(fmt.Sprintf("%d", i))), Fee: i, SizeInBytes: i, - }}, []*subtreepkg.TxInpoints{{ - ParentTxHashes: []chainhash.Hash{genesisHash}, - Idxs: [][]uint32{{uint32(i)}}, // Different output index for each tx - }}) + }}, []*subtreepkg.TxInpoints{singleParentInpointsPtr(genesisHash, uint32(i))}) } require.Eventually(t, func() bool { @@ -299,7 +298,15 @@ func setup(t *testing.T) (*BlockAssembly, *memory.Memory, *subtreepkg.Subtree, s txHash := chainhash.HashH([]byte(fmt.Sprintf("tx%d", i))) _ = subtree.AddNode(txHash, i, i) - txMap.Set(txHash, &subtreepkg.TxInpoints{ParentTxHashes: []chainhash.Hash{previousHash}, Idxs: [][]uint32{{0, 1}}}) + in0 := &bt.Input{PreviousTxOutIndex: 0} + require.NoError(t, in0.PreviousTxIDAdd(&previousHash)) + in1 := &bt.Input{PreviousTxOutIndex: 1} + require.NoError(t, in1.PreviousTxIDAdd(&previousHash)) + + ti, err := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{in0, in1}) + require.NoError(t, err) + + txMap.Set(txHash, &ti) previousHash = txHash } diff --git a/services/blockassembly/subtreeprocessor/SubtreeProcessor_test.go b/services/blockassembly/subtreeprocessor/SubtreeProcessor_test.go index 495af3fd29..7a71d9a1ed 100644 --- a/services/blockassembly/subtreeprocessor/SubtreeProcessor_test.go +++ b/services/blockassembly/subtreeprocessor/SubtreeProcessor_test.go @@ -2621,11 +2621,18 @@ func createSubtreeMeta(t *testing.T, subtree *subtreepkg.Subtree) *subtreepkg.Me parent := chainhash.HashH([]byte("txInpoints")) + parentInput := &bt.Input{PreviousTxOutIndex: 1} + if err := parentInput.PreviousTxIDAdd(&parent); err != nil { + panic(err) + } + + ti, err := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{parentInput}) + if err != nil { + panic(err) + } + for idx := range subtree.Nodes { - _ = subtreeMeta.SetTxInpoints(idx, subtreepkg.TxInpoints{ - ParentTxHashes: []chainhash.Hash{parent}, - Idxs: [][]uint32{{1}}, - }) + _ = subtreeMeta.SetTxInpoints(idx, ti) } return subtreeMeta diff --git a/services/blockassembly/subtreeprocessor/conflicting_queue_race_test.go b/services/blockassembly/subtreeprocessor/conflicting_queue_race_test.go index 8ef2e873de..0ad4f60a33 100644 --- a/services/blockassembly/subtreeprocessor/conflicting_queue_race_test.go +++ b/services/blockassembly/subtreeprocessor/conflicting_queue_race_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" subtreepkg "github.com/bsv-blockchain/go-subtree" blob_memory "github.com/bsv-blockchain/teranode/stores/blob/memory" @@ -52,16 +53,20 @@ func TestDequeueDuringBlockMovement_RejectsChildOfConflictingParent(t *testing.T childHash := chainhash.HashH([]byte("child-of-conflicting-parent")) otherHash := chainhash.HashH([]byte("unrelated-tx")) - childNode := subtreepkg.Node{Hash: childHash, Fee: 1, SizeInBytes: 250} - childInpoints := &subtreepkg.TxInpoints{ - ParentTxHashes: []chainhash.Hash{parentHash}, - Idxs: [][]uint32{{0}}, + mkInpoints := func(parent chainhash.Hash) *subtreepkg.TxInpoints { + in := &bt.Input{PreviousTxOutIndex: 0} + require.NoError(t, in.PreviousTxIDAdd(&parent)) + + ti, err := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{in}) + require.NoError(t, err) + + return &ti } + + childNode := subtreepkg.Node{Hash: childHash, Fee: 1, SizeInBytes: 250} + childInpoints := mkInpoints(parentHash) otherNode := subtreepkg.Node{Hash: otherHash, Fee: 2, SizeInBytes: 220} - otherInpoints := &subtreepkg.TxInpoints{ - ParentTxHashes: []chainhash.Hash{chainhash.HashH([]byte("unrelated-parent"))}, - Idxs: [][]uint32{{0}}, - } + otherInpoints := mkInpoints(chainhash.HashH([]byte("unrelated-parent"))) // Pin both clocks to the same instant so this test is deterministic // (no wall-time waits). With DoubleSpendWindow=0 the queue filter is diff --git a/services/blockassembly/subtreeprocessor/disk_tx_map_test.go b/services/blockassembly/subtreeprocessor/disk_tx_map_test.go index 7bde6f7812..793f3986ae 100644 --- a/services/blockassembly/subtreeprocessor/disk_tx_map_test.go +++ b/services/blockassembly/subtreeprocessor/disk_tx_map_test.go @@ -4,6 +4,7 @@ import ( "sync" "testing" + "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" subtreepkg "github.com/bsv-blockchain/go-subtree" "github.com/stretchr/testify/require" @@ -158,9 +159,18 @@ func TestDiskTxMap_ConcurrentSetIfNotExists(t *testing.T) { } func TestDiskTxMap_SerializationRoundtrip(t *testing.T) { - ip := makeInpoints(42) - ip.ParentTxHashes = append(ip.ParentTxHashes, chainhash.HashH([]byte("parent1"))) - ip.Idxs = append(ip.Idxs, []uint32{0, 1}) + parent := chainhash.HashH([]byte("parent1")) + + in0 := &bt.Input{PreviousTxOutIndex: 0} + require.NoError(t, in0.PreviousTxIDAdd(&parent)) + in1 := &bt.Input{PreviousTxOutIndex: 1} + require.NoError(t, in1.PreviousTxIDAdd(&parent)) + + built, err := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{in0, in1}) + require.NoError(t, err) + + built.SubtreeIndex = 42 + ip := &built serialized := serializeTxMapValue(ip) deserialized := deserializeTxMapValue(serialized) diff --git a/services/validator/Server_coverage_test.go b/services/validator/Server_coverage_test.go index 70417d9f94..60fe92a0a1 100644 --- a/services/validator/Server_coverage_test.go +++ b/services/validator/Server_coverage_test.go @@ -12,7 +12,6 @@ import ( "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" - "github.com/bsv-blockchain/go-subtree" "github.com/bsv-blockchain/teranode/errors" "github.com/bsv-blockchain/teranode/services/blockassembly" "github.com/bsv-blockchain/teranode/services/blockchain" @@ -220,7 +219,7 @@ func TestServerValidateTransaction(t *testing.T) { return &meta.Data{ Fee: 32279815860, SizeInBytes: 245, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*txid}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpoints(txid, 0), }, nil }, } @@ -266,7 +265,7 @@ func TestServerValidateTransaction(t *testing.T) { return &meta.Data{ Fee: 32279815860, SizeInBytes: 245, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*txid}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpoints(txid, 0), }, nil }, } @@ -328,7 +327,7 @@ func TestServerValidateTransactionBatch(t *testing.T) { return &meta.Data{ Fee: 32279815860, SizeInBytes: 245, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*txid}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpoints(txid, 0), }, nil }, } @@ -363,7 +362,7 @@ func TestServerValidateTransactionBatch(t *testing.T) { return &meta.Data{ Fee: 32279815860, SizeInBytes: 245, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*txid}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpoints(txid, 0), }, nil } return nil, errors.NewServiceError("validation error") diff --git a/services/validator/Server_test.go b/services/validator/Server_test.go index b20026eaef..2ad7119adb 100644 --- a/services/validator/Server_test.go +++ b/services/validator/Server_test.go @@ -27,6 +27,23 @@ var ( sampleTx, _ = hex.DecodeString("01000000016ec78dc364a3a911b38b32e58e5c228030f172da7d550e699939f6f3771f7f63010000006b483045022100dc6378291c4f8e06a54b0b60701cff7719c985db20537cd16d0350abe2f749660220694d67764dd8501e32190e6209a48103e3c7b202bb8c7682fc4c86e890c58409412103c3e59e22c5a32e54183a43993e8f584f709785f440fbf6f960551cb32041c5a9ffffffff03d0070000000000001976a9149e10b4a781c5be9f67367c3994fb3419aafd358e88ac2a240c00000000001976a914c52f8797b57f0b0cfc5856a5dd4a6f491a41822c88ac00000000000000000a006a075354554b2e434f00000000") ) +// singleParentInpoints builds a TxInpoints with one parent hash and one vout. +// Replaces the pre-packed-layout `subtree.TxInpoints{ParentTxHashes: +// []chainhash.Hash{*txid}, Idxs: [][]uint32{{0}}}` struct-literal pattern. +func singleParentInpoints(parent *chainhash.Hash, vout uint32) subtree.TxInpoints { + in := &bt.Input{PreviousTxOutIndex: vout} + if err := in.PreviousTxIDAdd(parent); err != nil { + panic(err) + } + + ti, err := subtree.NewTxInpointsFromInputs([]*bt.Input{in}) + if err != nil { + panic(err) + } + + return ti +} + func TestHTTPServer_Endpoints(t *testing.T) { // Create test context ctx := context.Background() @@ -56,7 +73,7 @@ func TestHTTPServer_Endpoints(t *testing.T) { return &meta.Data{ Fee: 32279815860, SizeInBytes: 245, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*txid}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpoints(txid, 0), }, nil }, } @@ -194,7 +211,7 @@ func TestValidatorHTTP_Endpoints(t *testing.T) { return &meta.Data{ Fee: 32279815860, SizeInBytes: 245, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*txid}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpoints(txid, 0), }, nil }, } @@ -303,7 +320,7 @@ func TestHTTPServerIntegration(t *testing.T) { return &meta.Data{ Fee: 32279815860, SizeInBytes: 245, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*txid}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpoints(txid, 0), }, nil }, } @@ -346,7 +363,7 @@ func TestHTTPServerHandlers(t *testing.T) { return &meta.Data{ Fee: 32279815860, SizeInBytes: 245, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*txid}, Idxs: [][]uint32{{0}}}, + TxInpoints: singleParentInpoints(txid, 0), }, nil }, } diff --git a/stores/txmetacache/txmetacache_test.go b/stores/txmetacache/txmetacache_test.go index 5920846113..0988d91385 100644 --- a/stores/txmetacache/txmetacache_test.go +++ b/stores/txmetacache/txmetacache_test.go @@ -138,8 +138,12 @@ func Test_txMetaCache_GetMeta(t *testing.T) { assert.Nil(t, metaGet.Tx) // Tx should be nil as it is not set in the cache assert.Equal(t, len(metaData.TxInpoints.ParentTxHashes), len(metaGet.TxInpoints.ParentTxHashes)) assert.Equal(t, metaData.TxInpoints.ParentTxHashes[0], metaGet.TxInpoints.ParentTxHashes[0]) - assert.Equal(t, len(metaData.TxInpoints.Idxs), len(metaGet.TxInpoints.Idxs)) - assert.Equal(t, metaData.TxInpoints.Idxs[0], metaGet.TxInpoints.Idxs[0]) + + origVouts, err := metaData.TxInpoints.GetParentVoutsAtIndex(0) + require.NoError(t, err) + gotVouts, err := metaGet.TxInpoints.GetParentVoutsAtIndex(0) + require.NoError(t, err) + assert.Equal(t, origVouts, gotVouts) }) } @@ -828,10 +832,15 @@ func Test_TxMetaCache_BatchDecorate(t *testing.T) { hash2, _ := chainhash.NewHashFromStr("b6fa2d4d23292bef7e13ffbb8c03168c97c457e1681642bf49b3e2ba7d26bb89") // Create test metadata + in1 := &bt.Input{PreviousTxOutIndex: 0} + require.NoError(t, in1.PreviousTxIDAdd(hash1)) + ti1, err := subtree.NewTxInpointsFromInputs([]*bt.Input{in1}) + require.NoError(t, err) + testMeta1 := &meta.Data{ Fee: 100, SizeInBytes: 250, - TxInpoints: subtree.TxInpoints{ParentTxHashes: []chainhash.Hash{*hash1}, Idxs: [][]uint32{{0}}}, + TxInpoints: ti1, BlockIDs: []uint32{1}, } diff --git a/stores/utxo/meta/data_test.go b/stores/utxo/meta/data_test.go index 45e8a61300..18976bf0da 100644 --- a/stores/utxo/meta/data_test.go +++ b/stores/utxo/meta/data_test.go @@ -13,23 +13,57 @@ import ( var ( hash3, _ = chainhash.NewHashFromStr("0ab59604a1c249d0cbfe18f01fe423df3035840f9a609395ccd177d2b217cae6") hash4, _ = chainhash.NewHashFromStr("08c3d6e8388415d8f6190a40c0acb9328b41a89a5854468e62c2bbd1dc740460") + + // testInpointsHash3Hash4 is the deduplicated TxInpoints with vouts {1, 2} + // for hash3 and {3, 4} for hash4 — built once and shared across tests. + testInpointsHash3Hash4 = mustInpoints(hash3, []uint32{1, 2}, hash4, []uint32{3, 4}) ) +// mustInpoints builds a TxInpoints from alternating (parentHash, vouts) pairs +// using fake *bt.Input values. Replaces the pre-packed-layout +// `subtree.TxInpoints{ParentTxHashes: ..., Idxs: ...}` struct-literal pattern. +func mustInpoints(args ...interface{}) subtree.TxInpoints { + inputs := make([]*bt.Input, 0) + + for i := 0; i < len(args); i += 2 { + parent := args[i].(*chainhash.Hash) + vouts := args[i+1].([]uint32) + + for _, v := range vouts { + in := &bt.Input{PreviousTxOutIndex: v} + if err := in.PreviousTxIDAdd(parent); err != nil { + panic(err) + } + + inputs = append(inputs, in) + } + } + + ti, err := subtree.NewTxInpointsFromInputs(inputs) + if err != nil { + panic(err) + } + + return ti +} + +// inpointsVouts returns the vouts for the i-th parent of a TxInpoints. Used +// in tests that previously read `txInpoints.Idxs[i]` directly. +func inpointsVouts(ti subtree.TxInpoints, i int) []uint32 { + v, err := ti.GetParentVoutsAtIndex(i) + if err != nil { + panic(err) + } + + return v +} + func Test_NewDataFromBytes(t *testing.T) { t.Run("test simple", func(t *testing.T) { data := &Data{ Fee: 100, SizeInBytes: 200, - TxInpoints: subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{ - *hash3, - *hash4, - }, - Idxs: [][]uint32{ - {1, 2}, - {3, 4}, - }, - }, + TxInpoints: testInpointsHash3Hash4, BlockIDs: []uint32{ 123, 321, @@ -52,8 +86,8 @@ func Test_NewDataFromBytes(t *testing.T) { require.Equal(t, len(data.TxInpoints.ParentTxHashes), len(d.TxInpoints.ParentTxHashes)) assert.Equal(t, data.TxInpoints.ParentTxHashes[0].String(), d.TxInpoints.ParentTxHashes[0].String()) assert.Equal(t, data.TxInpoints.ParentTxHashes[1].String(), d.TxInpoints.ParentTxHashes[1].String()) - assert.Equal(t, data.TxInpoints.Idxs[0], d.TxInpoints.Idxs[0]) - assert.Equal(t, data.TxInpoints.Idxs[1], d.TxInpoints.Idxs[1]) + assert.Equal(t, inpointsVouts(data.TxInpoints, 0), inpointsVouts(d.TxInpoints, 0)) + assert.Equal(t, inpointsVouts(data.TxInpoints, 1), inpointsVouts(d.TxInpoints, 1)) require.Len(t, data.BlockIDs, 2) require.Equal(t, len(data.BlockIDs), len(d.BlockIDs)) @@ -65,16 +99,7 @@ func Test_NewDataFromBytes(t *testing.T) { data := &Data{ Fee: 100, SizeInBytes: 200, - TxInpoints: subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{ - *hash3, - *hash4, - }, - Idxs: [][]uint32{ - {1, 2}, - {3, 4}, - }, - }, + TxInpoints: testInpointsHash3Hash4, BlockIDs: []uint32{ 123, 321, @@ -98,24 +123,15 @@ func Test_NewDataFromBytes(t *testing.T) { require.Equal(t, len(data.TxInpoints.ParentTxHashes), len(d.TxInpoints.ParentTxHashes)) assert.Equal(t, data.TxInpoints.ParentTxHashes[0].String(), d.TxInpoints.ParentTxHashes[0].String()) assert.Equal(t, data.TxInpoints.ParentTxHashes[1].String(), d.TxInpoints.ParentTxHashes[1].String()) - assert.Equal(t, data.TxInpoints.Idxs[0], d.TxInpoints.Idxs[0]) - assert.Equal(t, data.TxInpoints.Idxs[1], d.TxInpoints.Idxs[1]) + assert.Equal(t, inpointsVouts(data.TxInpoints, 0), inpointsVouts(d.TxInpoints, 0)) + assert.Equal(t, inpointsVouts(data.TxInpoints, 1), inpointsVouts(d.TxInpoints, 1)) }) t.Run("test frozen conflicting", func(t *testing.T) { data := &Data{ Fee: 100, SizeInBytes: 200, - TxInpoints: subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{ - *hash3, - *hash4, - }, - Idxs: [][]uint32{ - {1, 2}, - {3, 4}, - }, - }, + TxInpoints: testInpointsHash3Hash4, BlockIDs: []uint32{ 123, 321, @@ -154,16 +170,7 @@ func Benchmark_NewMetaDataFromBytes(b *testing.B) { data := &Data{ Fee: 100, SizeInBytes: 200, - TxInpoints: subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{ - *hash3, - *hash4, - }, - Idxs: [][]uint32{ - {1, 2}, - {3, 4}, - }, - }, + TxInpoints: testInpointsHash3Hash4, BlockIDs: []uint32{ 5, 6, @@ -185,16 +192,7 @@ func Benchmark_Bytes(b *testing.B) { data := &Data{ Fee: 100, SizeInBytes: 200, - TxInpoints: subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{ - *hash3, - *hash4, - }, - Idxs: [][]uint32{ - {1, 2}, - {3, 4}, - }, - }, + TxInpoints: testInpointsHash3Hash4, BlockIDs: []uint32{ 5, 6, @@ -214,16 +212,7 @@ func Benchmark_MetaBytes(b *testing.B) { data := &Data{ Fee: 100, SizeInBytes: 200, - TxInpoints: subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{ - *hash3, - *hash4, - }, - Idxs: [][]uint32{ - {1, 2}, - {3, 4}, - }, - }, + TxInpoints: testInpointsHash3Hash4, } b.ReportAllocs() diff --git a/stores/utxo/unmined_serialization_test.go b/stores/utxo/unmined_serialization_test.go index fa9a88c65f..671f594aef 100644 --- a/stores/utxo/unmined_serialization_test.go +++ b/stores/utxo/unmined_serialization_test.go @@ -3,6 +3,7 @@ package utxo import ( "testing" + "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/go-subtree" "github.com/stretchr/testify/assert" @@ -17,10 +18,13 @@ func TestSerializeDeserialize_Roundtrip(t *testing.T) { parentHash, err := chainhash.NewHashFromStr("0000000000000000000000000000000000000000000000000000000000000002") require.NoError(t, err) - txInpoints := &subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{*parentHash}, - Idxs: [][]uint32{{0}}, // Must match ParentTxHashes length - } + parentInput := &bt.Input{PreviousTxOutIndex: 0} + require.NoError(t, parentInput.PreviousTxIDAdd(parentHash)) + + txInpointsVal, err := subtree.NewTxInpointsFromInputs([]*bt.Input{parentInput}) + require.NoError(t, err) + + txInpoints := &txInpointsVal original := &UnminedTransaction{ Node: &subtree.Node{ @@ -179,11 +183,21 @@ func TestSerializeDeserialize_MultipleParents(t *testing.T) { parent3, err := chainhash.NewHashFromStr("0000000000000000000000000000000000000000000000000000000000000009") require.NoError(t, err) - txInpoints := &subtree.TxInpoints{ - ParentTxHashes: []chainhash.Hash{*parent1, *parent2, *parent3}, - Idxs: [][]uint32{{0}, {1}, {2}}, // Must match ParentTxHashes length + mkInput := func(parent *chainhash.Hash, vout uint32) *bt.Input { + in := &bt.Input{PreviousTxOutIndex: vout} + require.NoError(t, in.PreviousTxIDAdd(parent)) + return in } + txInpointsVal2, err := subtree.NewTxInpointsFromInputs([]*bt.Input{ + mkInput(parent1, 0), + mkInput(parent2, 1), + mkInput(parent3, 2), + }) + require.NoError(t, err) + + txInpoints := &txInpointsVal2 + original := &UnminedTransaction{ Node: &subtree.Node{ Hash: *hash, diff --git a/test/longtest/model/BlockBig_test.go b/test/longtest/model/BlockBig_test.go index 9d35bb8520..9801c52b80 100644 --- a/test/longtest/model/BlockBig_test.go +++ b/test/longtest/model/BlockBig_test.go @@ -424,7 +424,7 @@ func TestBigBlock_Valid(t *testing.T) { require.Equal(t, &meta.Data{ Fee: 1, SizeInBytes: 1, - TxInpoints: subtreepkg.TxInpoints{ParentTxHashes: nil, Idxs: nil}, + TxInpoints: subtreepkg.TxInpoints{}, BlockIDs: []uint32{}, }, data) diff --git a/test/longtest/services/blockassembly/Server_test.go b/test/longtest/services/blockassembly/Server_test.go index cfaa6d3afc..01742211d9 100644 --- a/test/longtest/services/blockassembly/Server_test.go +++ b/test/longtest/services/blockassembly/Server_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/bsv-blockchain/go-bt/v2" "github.com/bsv-blockchain/go-bt/v2/chainhash" subtreepkg "github.com/bsv-blockchain/go-subtree" "github.com/bsv-blockchain/teranode/services/blockassembly/blockassembly_api" @@ -54,10 +55,11 @@ func TestServer_Performance_1_million_txs_1_by_1(t *testing.T) { go func(bytesN []byte) { defer wg.Done() - txInpoints := subtreepkg.TxInpoints{ - ParentTxHashes: []chainhash.Hash{{}}, - Idxs: [][]uint32{{0}}, - } + emptyParent := chainhash.Hash{} + emptyParentInput := &bt.Input{PreviousTxOutIndex: 0} + _ = emptyParentInput.PreviousTxIDAdd(&emptyParent) + + txInpoints, _ := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{emptyParentInput}) txInpointsBytes, _ := txInpoints.Serialize() txid := make([]byte, 32) @@ -120,10 +122,11 @@ func TestServer_Performance_1_million_txs_1_by_1_with_sync_pool(t *testing.T) { go func(bytesN []byte) { defer wg.Done() - txInpoints := subtreepkg.TxInpoints{ - ParentTxHashes: []chainhash.Hash{{}}, - Idxs: [][]uint32{{0}}, - } + emptyParent := chainhash.Hash{} + emptyParentInput := &bt.Input{PreviousTxOutIndex: 0} + _ = emptyParentInput.PreviousTxIDAdd(&emptyParent) + + txInpoints, _ := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{emptyParentInput}) txInpointsBytes, _ := txInpoints.Serialize() txid := make([]byte, 32) @@ -182,10 +185,11 @@ func TestServer_Performance_1_million_txs_in_batches(t *testing.T) { go func() { defer wg.Done() - txInpoints := subtreepkg.TxInpoints{ - ParentTxHashes: []chainhash.Hash{{}}, - Idxs: [][]uint32{{0}}, - } + emptyParent := chainhash.Hash{} + emptyParentInput := &bt.Input{PreviousTxOutIndex: 0} + _ = emptyParentInput.PreviousTxIDAdd(&emptyParent) + + txInpoints, _ := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{emptyParentInput}) txInpointsBytes, _ := txInpoints.Serialize() requests := make([]*blockassembly_api.AddTxRequest, 0, 1_024) @@ -242,10 +246,11 @@ func TestServer_Performance_1_million_txs_in_batches_with_sync_pool(t *testing.T go func() { defer wg.Done() - txInpoints := subtreepkg.TxInpoints{ - ParentTxHashes: []chainhash.Hash{{}}, - Idxs: [][]uint32{{0}}, - } + emptyParent := chainhash.Hash{} + emptyParentInput := &bt.Input{PreviousTxOutIndex: 0} + _ = emptyParentInput.PreviousTxIDAdd(&emptyParent) + + txInpoints, _ := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{emptyParentInput}) txInpointsBytes, _ := txInpoints.Serialize() requests := make([]*blockassembly_api.AddTxRequest, 0, 1_024) diff --git a/test/longtest/services/blockassembly/subtreeprocessor/SubtreeProcessor_test.go b/test/longtest/services/blockassembly/subtreeprocessor/SubtreeProcessor_test.go index 3aab20d7a8..460523be61 100644 --- a/test/longtest/services/blockassembly/subtreeprocessor/SubtreeProcessor_test.go +++ b/test/longtest/services/blockassembly/subtreeprocessor/SubtreeProcessor_test.go @@ -279,7 +279,19 @@ func initMoveBlock(t *testing.T) (*subtreeprocessor.SubtreeProcessor, *memory.Me SizeInBytes: 1, } - stp.AddBatch([]subtreepkg.Node{node}, []*subtreepkg.TxInpoints{{ParentTxHashes: []chainhash.Hash{hash1, hash2}, Idxs: [][]uint32{{0, 1}, {2, 3}}}}) + mkIn := func(parent *chainhash.Hash, vout uint32) *bt.Input { + in := &bt.Input{PreviousTxOutIndex: vout} + require.NoError(t, in.PreviousTxIDAdd(parent)) + return in + } + + ti, err := subtreepkg.NewTxInpointsFromInputs([]*bt.Input{ + mkIn(&hash1, 0), mkIn(&hash1, 1), + mkIn(&hash2, 2), mkIn(&hash2, 3), + }) + require.NoError(t, err) + + stp.AddBatch([]subtreepkg.Node{node}, []*subtreepkg.TxInpoints{&ti}) } waitForSubtreeProcessorQueueToEmpty(t, stp)