impl missing parts of protocol 1.6: order mempool txs, headers list

src/electrumx/server/mempool.py

@@ -416,12 +416,15 @@ async def potential_spends(self, hashX):
         return result
 
     async def transaction_summaries(self, hashX):
-        '''Return a list of MemPoolTxSummary objects for the hashX.'''
+        '''Return a list of MemPoolTxSummary objects for the hashX,
+        sorted as expected by protocol methods.
+        '''
         result = []
         for tx_hash in self.hashXs.get(hashX, ()):
             tx = self.txs[tx_hash]
             has_ui = any(hash in self.txs for hash, idx in tx.prevouts)
             result.append(MemPoolTxSummary(tx_hash, tx.fee, has_ui))
+        result.sort(key=lambda x: (x.has_unconfirmed_inputs, x.hash[::-1]))
         return result
 
     async def unordered_UTXOs(self, hashX):

src/electrumx/server/session.py

@@ -1208,7 +1208,7 @@ async def address_status(self, hashX):
 
         Status is a hex string, but must be None if there is no history.
         '''
-        # Note history is ordered and mempool unordered in electrum-server
+        # Note both confirmed history and mempool history are ordered
         # For mempool, height is -1 if it has unconfirmed inputs, otherwise 0
         db_history, cost = await self.session_mgr.limited_history(hashX)
         mempool = await self.mempool.transaction_summaries(hashX)
@@ -1278,7 +1278,7 @@ async def scripthash_get_balance(self, scripthash):
         return await self.get_balance(hashX)
 
     async def unconfirmed_history(self, hashX):
-        # Note unconfirmed history is unordered in electrum-server
+        # Note both confirmed history and mempool history are ordered
         # height is -1 if it has unconfirmed inputs, otherwise 0
         result = [{'tx_hash': hash_to_hex_str(tx.hash),
                    'height': -tx.has_unconfirmed_inputs,
@@ -1288,7 +1288,7 @@ async def unconfirmed_history(self, hashX):
         return result
 
     async def confirmed_and_unconfirmed_history(self, hashX):
-        # Note history is ordered but unconfirmed is unordered in e-s
+        # Note both confirmed history and mempool history are ordered
         history, cost = await self.session_mgr.limited_history(hashX)
         self.bump_cost(cost)
         conf = [{'tx_hash': hash_to_hex_str(tx_hash), 'height': height}
@@ -1357,6 +1357,8 @@ async def block_headers(self, start_height, count, cp_height=0):
         start_height and count must be non-negative integers.  At most
         MAX_CHUNK_SIZE headers will be returned.
         '''
+        if self.protocol_tuple >= (1, 6):
+            return await self.block_headers_array(start_height, count, cp_height)
         start_height = non_negative_integer(start_height)
         count = non_negative_integer(count)
         cp_height = non_negative_integer(cp_height)
@@ -1373,6 +1375,38 @@ async def block_headers(self, start_height, count, cp_height=0):
         self.bump_cost(cost)
         return result
 
+    async def block_headers_array(self, start_height, count, cp_height=0):
+        '''Return block headers in an array for the main chain;
+        starting at start_height.
+        start_height and count must be non-negative integers.  At most
+        MAX_CHUNK_SIZE headers will be returned.
+        '''
+        start_height = non_negative_integer(start_height)
+        count = non_negative_integer(count)
+        cp_height = non_negative_integer(cp_height)
+        cost = count / 50
+
+        max_size = self.MAX_CHUNK_SIZE
+        count = min(count, max_size)
+        headers, count = await self.db.read_headers(start_height, count)
+        result = {'count': count, 'max': max_size, 'headers': []}
+        if count and cp_height:
+            cost += 1.0
+            last_height = start_height + count - 1
+            result.update(await self._merkle_proof(cp_height, last_height))
+
+        cursor = 0
+        height = 0
+        while cursor < len(headers):
+            next_cursor = self.db.header_offset(height + 1)
+            header = headers[cursor:next_cursor]
+            result['headers'].append(header.hex())
+            cursor = next_cursor
+            height += 1
+
+        self.bump_cost(cost)
+        return result
+
     def is_tor(self):
         '''Try to detect if the connection is to a tor hidden service we are
         running.'''
@@ -1960,36 +1994,41 @@ async def block_header(self, height, cp_height=0):
             return result
 
         # Covered by a checkpoint; truncate AuxPoW data
-        result['header'] = self.truncate_auxpow(result['header'], height)
+        result['header'] = self.truncate_auxpow_single(result['header'])
         return result
 
     async def block_headers(self, start_height, count, cp_height=0):
-        result = await super().block_headers(start_height, count, cp_height)
-
         # Older protocol versions don't truncate AuxPoW
         if self.protocol_tuple < (1, 4, 1):
-            return result
+            return await super().block_headers(start_height, count, cp_height)
 
         # Not covered by a checkpoint; return full AuxPoW data
         if cp_height == 0:
-            return result
+            return await super().block_headers(start_height, count, cp_height)
+
+        result = await super().block_headers_array(start_height, count, cp_height)
 
         # Covered by a checkpoint; truncate AuxPoW data
-        result['hex'] = self.truncate_auxpow(result['hex'], start_height)
-        return result
+        result['headers'] = self.truncate_auxpow_headers(result['headers'])
 
-    def truncate_auxpow(self, headers_full_hex, start_height):
-        height = start_height
-        headers_full = util.hex_to_bytes(headers_full_hex)
-        cursor = 0
-        headers = bytearray()
+        # Return headers in array form
+        if self.protocol_tuple >= (1, 6):
+            return result
 
-        while cursor < len(headers_full):
-            headers += headers_full[cursor:cursor+self.coin.TRUNCATED_HEADER_SIZE]
-            cursor += self.db.dynamic_header_len(height)
-            height += 1
+        # Return headers in concatenated form
+        result['hex'] = ''.join(result['headers'])
+        del result['headers']
+        return result
+
+    def truncate_auxpow_headers(self, headers):
+        result = []
+        for header in headers:
+            result.append(self.truncate_auxpow_single(header))
+        return result
 
-        return headers.hex()
+    def truncate_auxpow_single(self, header: str):
+        # 2 hex chars per byte
+        return header[:2*self.coin.TRUNCATED_HEADER_SIZE]
 
 
 class NameIndexElectrumX(ElectrumX):