stored_dict: perform type conversions at request time

this restores DB upgrades.
(DB upgrades require not converting stored classes)

Author: ecdsa

electrum/invoices.py

@@ -236,7 +236,7 @@ def get_id(self) -> str:
         else:  # on-chain
             return get_id_from_onchain_outputs(outputs=self.get_outputs(), timestamp=self.time)
 
-    def as_dict(self, status):
+    def export(self, status):
         d = {
             'is_lightning': self.is_lightning(),
             'amount_BTC': format_satoshis(self.get_amount_sat()),
@@ -298,7 +298,7 @@ def can_be_paid_onchain(self) -> bool:
             return True
 
     def to_debug_json(self) -> Dict[str, Any]:
-        d = self.to_json()
+        d = self.as_dict()
         d["lnaddr"] = self._lnaddr.to_debug_json()
         return d
 

electrum/json_db.py

@@ -30,9 +30,9 @@
 import jsonpatch
 import jsonpointer
 
-from .util import WalletFileException, profiler, sticky_property, MyEncoder
+from .util import WalletFileException, profiler, sticky_property
 from .logging import Logger
-from .stored_dict import _FLEX_KEY, BaseDB, _convert_dict_key, _convert_dict_value
+from .stored_dict import _FLEX_KEY, BaseDB
 from .storage import FileStorage
 
 
@@ -89,12 +89,10 @@ def __init__(
             *,
             allow_partial_writes = True,
             init_db = True,
-            encoder = MyEncoder,
     ):
         BaseDB.__init__(self, path)
         self._is_closed = True
         self.lock = threading.RLock()
-        self.encoder = encoder
         self.pending_changes = []  # type: List[str]
         self._modified = False
         if self.path:
@@ -105,18 +103,17 @@ def __init__(
                 self.init_db()
         else:
             self.storage = None
-            self.set_data('{}')
+            self.json_data = {}
             self._is_closed = False
 
     def set_data(self, json_str):
-        data = self.load_data(json_str)
-        self.json_data = self._convert_dict([], data)
+        self.json_data = self.load_data(json_str)
 
     def init_db(self):
         if self.storage.is_encrypted():
             assert self.storage.is_past_initial_decryption()
         json_str = self.storage.read()
-        self.set_data(json_str)
+        self.json_data = self.load_data(json_str)
         # write file in case there was a db upgrade
         self.write_and_force_consolidation()
         self._is_closed = False
@@ -286,7 +283,7 @@ def modified(self):
 
     @locked
     def add_patch(self, patch):
-        self.pending_changes.append(json.dumps(patch, cls=self.encoder))
+        self.pending_changes.append(json.dumps(patch))
         self.set_modified(True)
 
     def db_add(self, path, key: _FLEX_KEY, value) -> None:
@@ -310,29 +307,8 @@ def dump(self, *, human_readable: bool = True) -> str:
             self.json_data,
             indent=4 if human_readable else None,
             sort_keys=bool(human_readable),
-            cls=self.encoder,
         )
 
-    def _convert_dict_key(self, path: List[str], key: str) -> _FLEX_KEY:
-        return _convert_dict_key(path, key)
-
-    def _convert_dict_value(self, path: List[str], v) -> Any:
-        v = _convert_dict_value(path, v)
-        if isinstance(v, dict):
-            v = self._convert_dict(path, v)
-        return v
-
-    def _convert_dict(self, path: List[str], data: dict):
-        # recursively convert json dict to StoredDict
-        assert all(isinstance(x, str) for x in path), repr(path)
-        d = {}
-        for k, v in list(data.items()):
-            child_path = path + [k]
-            k = self._convert_dict_key(path, k)
-            v = self._convert_dict_value(child_path, v)
-            d[k] = v
-        return d
-
     @locked
     def write(self):
         if not self.storage:

electrum/lnchannel.py

@@ -62,6 +62,7 @@
 from .lnutil import ChannelBackupStorage, ImportedChannelBackupStorage, OnchainChannelBackupStorage
 from .lnutil import format_short_channel_id
 from .fee_policy import FEERATE_PER_KW_MIN_RELAY_LIGHTNING
+from .stored_dict import stored_at
 
 if TYPE_CHECKING:
     from .lnworker import LNWallet

electrum/lnpeer.py

@@ -1191,7 +1191,7 @@ async def channel_establishment_flow(
             lnworker=self.lnworker,
             initial_feerate=feerate
         )
-        temp_chan.storage['funding_inputs'] = [txin.prevout.to_json() for txin in funding_tx.inputs()]
+        temp_chan.storage['funding_inputs'] = [txin.prevout for txin in funding_tx.inputs()]
         temp_chan.storage['has_onchain_backup'] = has_onchain_backup
         temp_chan.storage['init_height'] = self.lnworker.network.get_local_height()
         temp_chan.storage['init_timestamp'] = int(time.time())

electrum/lnutil.py

@@ -1942,7 +1942,7 @@ def from_tuple(amount_msat, rhash, cltv_abs, htlc_id, timestamp) -> 'UpdateAddHt
             htlc_id=htlc_id,
             timestamp=timestamp)
 
-    def to_json(self):
+    def as_tuple(self):
         self._validate()
         return dataclasses.astuple(self)
 

electrum/stored_dict.py

@@ -89,39 +89,29 @@ def decorator(func):
     return decorator
 
 
-def _walk_path(d, path):
-    for k in path:
-        if k in d:
-            d = d[k]
-        elif '*' in d:
-            d = d['*']
-        else:
-            return None
-    return d
-
-def _convert_dict_key(path: List[str], key: str) -> _FLEX_KEY:
-    """Maybe convert key from str to python type (typically int or IntEnum)"""
-    assert all(isinstance(x, str) for x in path), repr(path)
-    r = _walk_path(registered_keys, path)
-    if r:
-        if func := r.get('self'):
-            key = func(key)
-    assert isinstance(key, _FLEX_KEY), f"unexpected type for {key=!r} at {path=}"
-    return key
-
-def _convert_dict_value(path: List[str], v) -> Any:
-    assert all(isinstance(x, str) for x in path), repr(path)
-    r = _walk_path(registered_names, path)
-    if r and type(r) is tuple:
-        _type, constructor = r
-        if _type == dict:
-            v = constructor(**v)
-        elif _type == tuple:
-            v = constructor(*v)
-        else:
-            v = constructor(v)
-    return v
-
+def to_default(obj):
+    """Convert user-defined classes to python built-in types.
+    Also convert bytes to hex, so that the result is json serializable.
+    """
+    if obj is None or isinstance(obj, (str, int, float)):
+        return obj
+    if isinstance(obj, bytes):
+        return obj.hex()
+    if hasattr(obj, 'as_str') and callable(obj.as_str):
+        return obj.as_str()
+    if hasattr(obj, 'as_dict') and callable(obj.as_dict):
+        obj = obj.as_dict()
+    if hasattr(obj, 'as_tuple') and callable(obj.as_tuple):
+        obj = obj.as_tuple()
+    if isinstance(obj, (set, frozenset)):
+        return [to_default(x) for x in list(obj)]
+    if isinstance(obj, dict):
+        return dict([(key_to_str(k), to_default(v)) for k, v in obj.items()])
+    if isinstance(obj, list):
+        return [to_default(x) for x in obj]
+    if isinstance(obj, tuple):
+        return tuple([to_default(x) for x in list(obj)])
+    raise Exception('unsupported type', type(obj))
 
 
 
@@ -131,6 +121,7 @@ def __init__(self, path):
         Logger.__init__(self)
         self._write_batch = None
         self.path = path
+        self._should_convert = True
 
     def file_exists(self):
         raise NotImplementedError()
@@ -178,9 +169,9 @@ def _to_stored_dict_or_list(self, key, value):
             value = StoredList(self._db, key=key, parent=self)
         elif isinstance(value, dict):
             value = StoredDict(self._db, key=key, parent=self)
-        #elif isinstance(value, tuple):
-        #    value = StoredList(self._db, key=key, parent=self)
-        #    value = tuple(value[:]) # do not expose StoredTuple to callers
+        elif isinstance(value, tuple):
+            value = StoredList(self._db, key=key, parent=self)
+            value = tuple(value[:]) # do not expose StoredTuple to callers
         return value
 
     @property
@@ -193,12 +184,37 @@ def hint(self):
     def db_get(self, key):
         value = self._db.get(self.hint, key)
         value = self._to_stored_dict_or_list(key, value)
+        if not self.should_convert():
+            return value
+        value = self._convert_value(key, value)
         # set db for StoredObject, because it is not set in the constructor
         if isinstance(value, StoredObject):
             value.set_db(self._db)
             value.set_parent(key=key, parent=self)
         return value
 
+    def _convert_key(self, key: str) -> _FLEX_KEY:
+        """Maybe convert key from str to python type (typically int or IntEnum)"""
+        if self._key_converters:
+            if func := self._key_converters.get('self'):
+                key = func(key)
+        assert isinstance(key, _FLEX_KEY), f"unexpected type for {key=!r} at {self._path}"
+        return key
+
+    def _convert_value(self, key, v) -> Any:
+        reg = self.get_constructor(key)
+        if reg:
+            if isinstance(v, (StoredDict, StoredList)):
+                v = v.dump()
+            _type, constructor = reg
+            if _type == dict:
+                v = constructor(**v)
+            elif _type == tuple:
+                v = constructor(*v)
+            else:
+                v = constructor(v)
+        return v
+
     def get_constructor(self, key):
         if self._constructor:
             r = self._constructor.get(key, self._constructor.get('*', None))
@@ -245,10 +261,10 @@ def __setattr__(self, key: str, value):
         assert isinstance(key, str), repr(key)
         if not key.startswith('_') and self._path:
             if value != getattr(self, key):
-                self._db.replace(self.hint, self._path, key, value)
+                self._db.replace(self.hint, self._path, key, to_default(value))
         object.__setattr__(self, key, value)
 
-    def to_json(self):
+    def as_dict(self):
         d = dict(vars(self))
         # don't expose/store private stuff
         d = {k: v for k, v in d.items()
@@ -277,6 +293,9 @@ def __init__(self, db: BaseDB, key: _FLEX_KEY, parent):
         self.init_constructor()
         self.init_key_converters()
 
+    def should_convert(self):
+        return self._db._should_convert
+
     def dump(self) -> dict:
         data = {}
         for k, v in self.items():
@@ -286,18 +305,23 @@ def dump(self) -> dict:
         return data
 
     def __getitem__(self, key: _FLEX_KEY) -> Any:
+        key = key_to_str(key)
         return self.db_get(key)
 
     def __setitem__(self, key: _FLEX_KEY, value: Any) -> None:
+        key = key_to_str(key)
         if isinstance(value, StoredObject):
             # side effect
             value.set_db(self._db)
             value.set_parent(key=key, parent=self)
         if isinstance(value, (StoredList, StoredDict)):
             value = value.dump()
+        # convert to python
+        value = to_default(value)
         self._db.put(self.hint, self._path, key, value)
 
     def __delitem__(self, key: _FLEX_KEY) -> None:
+        key = key_to_str(key)
         self._db.remove(self.hint, self._path, key)
 
     def __iter__(self) -> Iterator[str]:
@@ -309,19 +333,20 @@ def __len__(self) -> int:
     # ---- Dict-like extras ----
 
     def __contains__(self, key: object) -> bool:
+        key = key_to_str(key)
         return self._db.dict_contains(self.hint, self._path, key)
 
     def keys(self) -> Iterable[str]:
         for k in self._db.iter_keys(self.hint, self._path):
-            yield k
+            yield self._convert_key(k)
 
     def values(self) -> Iterator[Any]:
         for k in self._db.iter_keys(self.hint, self._path):
             yield self[k]
 
     def items(self) -> Iterator[Tuple[str, Any]]:
         for k in self._db.iter_keys(self.hint, self._path):
-            yield (k, self[k])
+            yield (self._convert_key(k), self[k])
 
     def get(self, key: _FLEX_KEY, default: Any = None, add_if_missing=False) -> Any:
         # If add_if_missing is True, create DB entry if it does not exist.
@@ -383,6 +408,9 @@ def __init__(self, db: BaseDB, key: _FLEX_KEY, parent):
         self.init_constructor()
         self.init_key_converters()
 
+    def should_convert(self):
+        return self._db._should_convert
+
     def _get_list_item(self, key: int):
         key = int(key)
         return self.db_get(key)
@@ -408,16 +436,19 @@ def __iter__(self) -> Iterator[str]:
             yield self._get_list_item(i)
 
     def append(self, value):
+        value = to_default(value)
         self._db.list_append(self.hint, self._path, value)
 
     def clear(self):
         self._db.list_clear(self.hint, self._path)
         assert len(self) == 0
 
     def index(self, item) -> int:
+        item = to_default(item)
         return self._db.list_index(self.hint, self._path, item)
 
     def remove(self, item):
+        item = to_default(item)
         self._db.list_remove(self.hint, self._path, item)
 
     def dump(self) -> list:

electrum/transaction.py

@@ -164,6 +164,9 @@ def to_legacy_tuple(self) -> Tuple[int, str, Union[int, str]]:
             return TYPE_ADDRESS, self.address, self.value
         return TYPE_SCRIPT, self.scriptpubkey.hex(), self.value
 
+    def as_tuple(self):
+        return self.to_legacy_tuple()
+
     @classmethod
     def from_legacy_tuple(cls, _type: int, addr: str, val: Union[int, str]) -> Union['TxOutput', 'PartialTxOutput']:
         if _type == TYPE_ADDRESS:
@@ -305,8 +308,8 @@ def __repr__(self):
     def to_str(self) -> str:
         return f"{self.txid.hex()}:{self.out_idx}"
 
-    def to_json(self):
-        return [self.txid.hex(), self.out_idx]
+    def as_tuple(self):
+        return (self.txid.hex(), self.out_idx)
 
     def serialize_to_network(self) -> bytes:
         return self.txid[::-1] + int.to_bytes(self.out_idx, length=4, byteorder="little", signed=False)
@@ -907,6 +910,9 @@ class Transaction:
     def __str__(self):
         return self.serialize()
 
+    def as_str(self):
+        return str(self)
+
     def __init__(self, raw):
         if raw is None:
             self._cached_network_ser = None