From 269f2e0143f9e00fe7fa033f92e454a0f006960a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 15 Dec 2020 19:03:13 +0100 Subject: [PATCH 1/4] protocol change: define order of mempool txs in status of scripthash Note that this is a soft fork: the server can apply it even for past protocol versions. Previously, with the order being undefined, if an address had multiple mempool transactions touching it, switching between different servers could result in a change in address status simply as a result of these servers ordering mempool txs differently. This would result in the client re-requesting the whole history of the address. ----- D/i | interface.[electrum.blockstream.info:60002] | <-- ('blockchain.scripthash.subscribe', ['660b44502503064f9d5feee48726287c0973e25bc531b4b8a072f57f143d5cd0']) {} (id: 12) D/i | interface.[electrum.blockstream.info:60002] | --> 9da27f9df91e3f860212f65b736fa20a539ba6e3d509f6370367ee7f10a4d5b0 (id: 12) D/i | interface.[electrum.blockstream.info:60002] | <-- ('blockchain.scripthash.get_history', ['660b44502503064f9d5feee48726287c0973e25bc531b4b8a072f57f143d5cd0']) {} (id: 13) D/i | interface.[electrum.blockstream.info:60002] | --> [ {'fee': 200, 'height': 0, 'tx_hash': '3ee6d6e26291ce360127fe039b816470fce6eeea19b5c9d10829a1e4efc2d0c7'}, {'fee': 239, 'height': 0, 'tx_hash': '9e050f09b676b9b0ee26aa02ccee623fae585a85d6a5e24ecedd6f8d6d2d3b1d'}, {'fee': 178, 'height': 0, 'tx_hash': 'fb80adbf8274190418cb3fb0385d82fe9d47a844d9913684fa5fb3d48094b35a'}, {'fee': 200, 'height': 0, 'tx_hash': '713933c50b7c43f606dad5749ea46e3bc6622657e9b13ace9d639697da266e8b'} ] (id: 13) D/i | interface.[testnet.hsmiths.com:53012] | <-- ('blockchain.scripthash.subscribe', ['660b44502503064f9d5feee48726287c0973e25bc531b4b8a072f57f143d5cd0']) {} (id: 12) D/i | interface.[testnet.hsmiths.com:53012] | --> f7ef7237d2d62a3280acae05616200b96ad9dd85fd0473c29152a4a41e05686c (id: 12) D/i | interface.[testnet.hsmiths.com:53012] | <-- ('blockchain.scripthash.get_history', ['660b44502503064f9d5feee48726287c0973e25bc531b4b8a072f57f143d5cd0']) {} (id: 13) D/i | interface.[testnet.hsmiths.com:53012] | --> [ {'tx_hash': '9e050f09b676b9b0ee26aa02ccee623fae585a85d6a5e24ecedd6f8d6d2d3b1d', 'height': 0, 'fee': 239}, {'tx_hash': 'fb80adbf8274190418cb3fb0385d82fe9d47a844d9913684fa5fb3d48094b35a', 'height': 0, 'fee': 178}, {'tx_hash': '3ee6d6e26291ce360127fe039b816470fce6eeea19b5c9d10829a1e4efc2d0c7', 'height': 0, 'fee': 200}, {'tx_hash': '713933c50b7c43f606dad5749ea46e3bc6622657e9b13ace9d639697da266e8b', 'height': 0, 'fee': 200} ] (id: 13) --- src/electrumx/server/mempool.py | 5 ++++- src/electrumx/server/session.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/electrumx/server/mempool.py b/src/electrumx/server/mempool.py index d38f9a7ac..533233c1e 100644 --- a/src/electrumx/server/mempool.py +++ b/src/electrumx/server/mempool.py @@ -416,12 +416,15 @@ async def potential_spends(self, hashX): return result async def transaction_summaries(self, hashX): - '''Return a list of MemPoolTxSummary objects for the hashX.''' + '''Return a list of MemPoolTxSummary objects for the hashX, + sorted as expected by protocol methods. + ''' result = [] for tx_hash in self.hashXs.get(hashX, ()): tx = self.txs[tx_hash] has_ui = any(hash in self.txs for hash, idx in tx.prevouts) result.append(MemPoolTxSummary(tx_hash, tx.fee, has_ui)) + result.sort(key=lambda x: (x.has_unconfirmed_inputs, x.hash[::-1])) return result async def unordered_UTXOs(self, hashX): diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 496efd195..712c45f54 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -1208,7 +1208,7 @@ async def address_status(self, hashX): Status is a hex string, but must be None if there is no history. ''' - # Note history is ordered and mempool unordered in electrum-server + # Note both confirmed history and mempool history are ordered # For mempool, height is -1 if it has unconfirmed inputs, otherwise 0 db_history, cost = await self.session_mgr.limited_history(hashX) mempool = await self.mempool.transaction_summaries(hashX) @@ -1278,7 +1278,7 @@ async def scripthash_get_balance(self, scripthash): return await self.get_balance(hashX) async def unconfirmed_history(self, hashX): - # Note unconfirmed history is unordered in electrum-server + # Note both confirmed history and mempool history are ordered # height is -1 if it has unconfirmed inputs, otherwise 0 result = [{'tx_hash': hash_to_hex_str(tx.hash), 'height': -tx.has_unconfirmed_inputs, @@ -1288,7 +1288,7 @@ async def unconfirmed_history(self, hashX): return result async def confirmed_and_unconfirmed_history(self, hashX): - # Note history is ordered but unconfirmed is unordered in e-s + # Note both confirmed history and mempool history are ordered history, cost = await self.session_mgr.limited_history(hashX) self.bump_cost(cost) conf = [{'tx_hash': hash_to_hex_str(tx_hash), 'height': height} From 3d92d162054fc2e352694807c26602a7137cedd6 Mon Sep 17 00:00:00 2001 From: Ahmed Bodiwala Date: Sun, 28 Jun 2020 00:15:59 +0000 Subject: [PATCH 2/4] [AuxPow] Add Support for individual block headers instead of a combined hex string Array headers: fix type error Array headers: move variable initialization --- src/electrumx/server/session.py | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 712c45f54..0c1f5842c 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -1357,6 +1357,8 @@ async def block_headers(self, start_height, count, cp_height=0): start_height and count must be non-negative integers. At most MAX_CHUNK_SIZE headers will be returned. ''' + if self.protocol_tuple >= (1, 6): + return await self.block_headers_array(start_height, count, cp_height) start_height = non_negative_integer(start_height) count = non_negative_integer(count) cp_height = non_negative_integer(cp_height) @@ -1373,6 +1375,38 @@ async def block_headers(self, start_height, count, cp_height=0): self.bump_cost(cost) return result + async def block_headers_array(self, start_height, count, cp_height=0): + '''Return block headers in an array for the main chain; + starting at start_height. + start_height and count must be non-negative integers. At most + MAX_CHUNK_SIZE headers will be returned. + ''' + start_height = non_negative_integer(start_height) + count = non_negative_integer(count) + cp_height = non_negative_integer(cp_height) + cost = count / 50 + + max_size = self.MAX_CHUNK_SIZE + count = min(count, max_size) + headers, count = await self.db.read_headers(start_height, count) + result = {'count': count, 'max': max_size, 'headers': []} + if count and cp_height: + cost += 1.0 + last_height = start_height + count - 1 + result.update(await self._merkle_proof(cp_height, last_height)) + + cursor = 0 + height = 0 + while cursor < len(headers): + next_cursor = self.db.header_offset(height + 1) + header = headers[cursor:next_cursor] + result['headers'].append(header.hex()) + cursor = next_cursor + height += 1 + + self.bump_cost(cost) + return result + def is_tor(self): '''Try to detect if the connection is to a tor hidden service we are running.''' @@ -1975,9 +2009,19 @@ async def block_headers(self, start_height, count, cp_height=0): return result # Covered by a checkpoint; truncate AuxPoW data + if self.protocol_tuple >= (1, 6): + result['headers'] = self.truncate_auxpow_headers(result['headers']) + return + result['hex'] = self.truncate_auxpow(result['hex'], start_height) return result + def truncate_auxpow_headers(self, headers): + result = [] + for header in headers: + result.append(header[:self.coin.TRUNCATED_HEADER_SIZE]) + return result + def truncate_auxpow(self, headers_full_hex, start_height): height = start_height headers_full = util.hex_to_bytes(headers_full_hex) From d088c3e912346a5424e0a677b4d68d306ded7438 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Mar 2021 22:40:09 +0000 Subject: [PATCH 3/4] Array headers: Refactor AuxPoW --- src/electrumx/server/session.py | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 0c1f5842c..3ef354c13 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -1994,26 +1994,30 @@ async def block_header(self, height, cp_height=0): return result # Covered by a checkpoint; truncate AuxPoW data - result['header'] = self.truncate_auxpow(result['header'], height) + result['header'] = result['header'][:self.coin.TRUNCATED_HEADER_SIZE] return result async def block_headers(self, start_height, count, cp_height=0): - result = await super().block_headers(start_height, count, cp_height) - # Older protocol versions don't truncate AuxPoW if self.protocol_tuple < (1, 4, 1): - return result + return await super().block_headers(start_height, count, cp_height) # Not covered by a checkpoint; return full AuxPoW data if cp_height == 0: - return result + return await super().block_headers(start_height, count, cp_height) + + result = await super().block_headers_array(start_height, count, cp_height) # Covered by a checkpoint; truncate AuxPoW data + result['headers'] = self.truncate_auxpow_headers(result['headers']) + + # Return headers in array form if self.protocol_tuple >= (1, 6): - result['headers'] = self.truncate_auxpow_headers(result['headers']) - return + return result - result['hex'] = self.truncate_auxpow(result['hex'], start_height) + # Return headers in concatenated form + result['hex'] = ''.join(result['headers']) + del result['headers'] return result def truncate_auxpow_headers(self, headers): @@ -2022,19 +2026,6 @@ def truncate_auxpow_headers(self, headers): result.append(header[:self.coin.TRUNCATED_HEADER_SIZE]) return result - def truncate_auxpow(self, headers_full_hex, start_height): - height = start_height - headers_full = util.hex_to_bytes(headers_full_hex) - cursor = 0 - headers = bytearray() - - while cursor < len(headers_full): - headers += headers_full[cursor:cursor+self.coin.TRUNCATED_HEADER_SIZE] - cursor += self.db.dynamic_header_len(height) - height += 1 - - return headers.hex() - class NameIndexElectrumX(ElectrumX): def set_request_handlers(self, ptuple): From 7f8f82ffec03871d75dfb4214628c8332daa7b3a Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Mar 2021 23:41:20 +0000 Subject: [PATCH 4/4] Array headers: Refactor AuxPoW truncation --- src/electrumx/server/session.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/electrumx/server/session.py b/src/electrumx/server/session.py index 3ef354c13..a41dbf0ab 100644 --- a/src/electrumx/server/session.py +++ b/src/electrumx/server/session.py @@ -1994,7 +1994,7 @@ async def block_header(self, height, cp_height=0): return result # Covered by a checkpoint; truncate AuxPoW data - result['header'] = result['header'][:self.coin.TRUNCATED_HEADER_SIZE] + result['header'] = self.truncate_auxpow_single(result['header']) return result async def block_headers(self, start_height, count, cp_height=0): @@ -2023,9 +2023,13 @@ async def block_headers(self, start_height, count, cp_height=0): def truncate_auxpow_headers(self, headers): result = [] for header in headers: - result.append(header[:self.coin.TRUNCATED_HEADER_SIZE]) + result.append(self.truncate_auxpow_single(header)) return result + def truncate_auxpow_single(self, header: str): + # 2 hex chars per byte + return header[:2*self.coin.TRUNCATED_HEADER_SIZE] + class NameIndexElectrumX(ElectrumX): def set_request_handlers(self, ptuple):