Skip to content

Commit fa3bd95

Browse files
committed
db: unify everything to use big-endian encoding, not mixed
Performance-wise, LE was a tiny bit faster though not consistently. Perf goes 3-4% both ways, twice in LE's favour, once in BE's favour. Still, BE is nicer as that's the natural order for leveldb comparators. ``` $ python3 -m timeit -n1 -r10 --setup "import random; from electrumx.lib.util import unpack_be_uint64, unpack_le_uint64; arr = [random.randbytes(8) for _ in range(10**7)]" "[unpack_be_uint64(x) for x in arr]" 1 loop, best of 10: 1.12 sec per loop $ python3 -m timeit -n1 -r10 --setup "import random; from electrumx.lib.util import unpack_be_uint64, unpack_le_uint64; arr = [random.randbytes(8) for _ in range(10**7)]" "[unpack_le_uint64(x) for x in arr]" 1 loop, best of 10: 1.08 sec per loop $ python3 -m timeit -n1 -r10 --setup "import random; from electrumx.lib.util import pack_be_uint64, pack_le_uint64; arr = [int.from_bytes(random.randbytes(8), byteorder='big') for _ in range(10**7)]" "[pack_be_uint64(x) for x in arr]" 1 loop, best of 10: 683 msec per loop $ python3 -m timeit -n1 -r10 --setup "import random; from electrumx.lib.util import pack_be_uint64, pack_le_uint64; arr = [int.from_bytes(random.randbytes(8), byteorder='big') for _ in range(10**7)]" "[pack_le_uint64(x) for x in arr]" 1 loop, best of 10: 705 msec per loop ``` ``` $ python3 -m timeit -n1 -r10 --setup "import random; from electrumx.lib.util import unpack_be_uint32, unpack_le_uint32; arr = [random.randbytes(4) for _ in range(10**7)]" "[unpack_be_uint32(x) for x in arr]" 1 loop, best of 10: 1.03 sec per loop $ python3 -m timeit -n1 -r10 --setup "import random; from electrumx.lib.util import unpack_be_uint32, unpack_le_uint32; arr = [random.randbytes(4) for _ in range(10**7)]" "[unpack_le_uint32(x) for x in arr]" 1 loop, best of 10: 1.03 sec per loop $ python3 -m timeit -n1 -r10 --setup "import random; from electrumx.lib.util import pack_be_uint32, pack_le_uint32; arr = [int.from_bytes(random.randbytes(4), byteorder='big') for _ in range(10**7)]" "[pack_be_uint32(x) for x in arr]" 1 loop, best of 10: 629 msec per loop $ python3 -m timeit -n1 -r10 --setup "import random; from electrumx.lib.util import pack_be_uint32, pack_le_uint32; arr = [int.from_bytes(random.randbytes(4), byteorder='big') for _ in range(10**7)]" "[pack_le_uint32(x) for x in arr]" 1 loop, best of 10: 605 msec per loop ```
1 parent 1c98c7b commit fa3bd95

2 files changed

Lines changed: 15 additions & 9 deletions

File tree

src/electrumx/server/db.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ class DB:
7676
it was shutdown uncleanly.
7777
'''
7878

79-
DB_VERSIONS = (10, )
79+
DB_VERSIONS = (11, )
8080
DB_STATE_KEY = b'state'
8181

8282
utxo_db: Optional['Storage']

src/electrumx/server/db_util.py

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
from typing import Sequence
66

77
from electrumx.lib.util import (
8-
pack_le_uint32, unpack_le_uint32,
9-
pack_le_uint64, unpack_le_uint64,
108
pack_be_uint32, unpack_be_uint32,
119
pack_be_uint64, unpack_be_uint64,
1210
)
@@ -25,26 +23,34 @@ def pack_txnum(tx_num: int) -> bytes:
2523
return pack_be_uint64(tx_num)[-TXNUM_LEN:]
2624

2725

26+
assert pack_txnum(0x123) == bytes.fromhex("0000000123")
27+
assert unpack_txnum(pack_txnum(0x123)) == 0x123
28+
29+
2830
# txout_idx
2931
TXOUTIDX_LEN = 3
3032
TXOUTIDX_PADDING = bytes(4 - TXOUTIDX_LEN)
3133

3234

3335
def unpack_txoutidx(txout_idx: bytes) -> int:
34-
return unpack_le_uint32(txout_idx + TXOUTIDX_PADDING)[0]
36+
return unpack_be_uint32(TXOUTIDX_PADDING + txout_idx)[0]
3537

3638

3739
def pack_txoutidx(txout_idx: int) -> bytes:
38-
return pack_le_uint32(txout_idx)[:TXOUTIDX_LEN]
40+
return pack_be_uint32(txout_idx)[-TXOUTIDX_LEN:]
41+
42+
43+
assert pack_txoutidx(0x123) == bytes.fromhex("000123")
44+
assert unpack_txoutidx(pack_txoutidx(0x123)) == 0x123
3945

4046

4147
# sats
4248
def unpack_satoshis_val(sats: bytes) -> int:
43-
return unpack_le_uint64(sats)[0]
49+
return unpack_be_uint64(sats)[0]
4450

4551

4652
def pack_satoshis_val(sats: int) -> bytes:
47-
return pack_le_uint64(sats)
53+
return pack_be_uint64(sats)
4854

4955

5056
# block height
@@ -64,11 +70,11 @@ def pack_block_height(bheight: int) -> bytes:
6470

6571

6672
def unpack_dyn_header_offset(offset: bytes) -> int:
67-
return unpack_le_uint64(offset)[0]
73+
return unpack_be_uint64(offset)[0]
6874

6975

7076
def pack_dyn_header_offset(offset: int) -> bytes:
71-
return pack_le_uint64(offset)
77+
return pack_be_uint64(offset)
7278

7379

7480
class DBTooOldForMigrations(RuntimeError):

0 commit comments

Comments
 (0)