Fix regressions from ujson removal and add regression tests

Fixes layered on top of #1676's ujson removal. Dropping the dependency
resolves the four ujson CVEs affecting the previously pinned 1.33
(CVE-2022-31116, CVE-2022-31117, CVE-2026-44660, CVE-2026-54911;
CVE-2021-45958 cited in #1676 only affects ujson >= 1.34).

- channel.py: the lint fix in #1676 replaced `type(msg) != tuple` with
  `not isinstance(type(msg), tuple)`, which is always True and wrapped
  already-tuple messages into nested tuples, breaking handler routing.
  Restore the original semantics in a lint-clean form
  (`type(msg) is not tuple`), preserving NamedTuple wrapping.

- zstack serializeMsg: ujson emitted compact JSON; stdlib json's default
  separators add whitespace, growing every wire message and breaking the
  exact-size assertions in stp_zmq/test/test_zstack.py calibrated to
  MSG_LEN_LIMIT. Pass separators=(',', ':') to keep the wire format
  byte-compatible. (Applied to scripts/test_zmq's copy as well.)

- recorder: SimpleZStackWithRecorder records raw wire frames; ujson
  silently encoded bytes as UTF-8 strings while stdlib json raises
  TypeError, so recording mode (STACK_COMPANION=1) crashed on the first
  message. Add a shared bytes-decoding default hook
  (plenum.common.util.json_default_bytes_to_str) and a bytes regression
  test.

- Add regression tests pinning JsonSerializer's exact byte output
  (key ordering, compact separators, raw-UTF-8 non-ASCII, float repr,
  int-key coercion, top-level-bytes base64) since it feeds
  signing/hashing, plus characterisation of the nested-bytes TypeError.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: kukgini <kukgini@gmail.com>

Author: kukgini

common/test/test_json_serializer_ujson_compat.py

@@ -0,0 +1,73 @@
+"""
+Regression tests added when the ``ujson`` dependency was removed: pin the
+exact byte output of ``JsonSerializer`` (sorted keys, compact separators,
+raw UTF-8), since it feeds signing/hashing and must stay byte-stable.
+"""
+import pytest
+
+from common.serializers.json_serializer import JsonSerializer
+
+
+@pytest.fixture
+def sz():
+    return JsonSerializer()
+
+
+def test_keys_sorted_and_compact(sz):
+    # Insertion order must not affect output; keys are sorted, no whitespace.
+    assert sz.serialize({'b': 2, 'a': 1, 'c': 3}, toBytes=False) == '{"a":1,"b":2,"c":3}'
+
+
+def test_non_ascii_kept_raw_utf8(sz):
+    # ensure_ascii=False: characters are emitted as raw UTF-8, not \\uXXXX.
+    assert sz.serialize({'name': 'héllo', 'kr': '한글', 'emoji': '🚀'},
+                        toBytes=False) == '{"emoji":"🚀","kr":"한글","name":"héllo"}'
+    # And the bytes form is the UTF-8 encoding of that string.
+    assert sz.serialize({'name': 'héllo'}) == '{"name":"héllo"}'.encode('utf-8')
+
+
+def test_float_repr_pinned(sz):
+    assert sz.serialize({'a': 14.8639, 'b': -97.466179, 'c': 1.0, 'd': 1e20},
+                        toBytes=False) == '{"a":14.8639,"b":-97.466179,"c":1.0,"d":1e+20}'
+
+
+def test_int_keys_become_strings(sz):
+    # json coerces non-string keys to strings and then sorts lexicographically.
+    assert sz.serialize({3: 'c', 1: 'a', 2: 'b'}, toBytes=False) == '{"1":"a","2":"b","3":"c"}'
+
+
+def test_bool_and_none(sz):
+    assert sz.serialize({'t': True, 'f': False, 'n': None},
+                        toBytes=False) == '{"f":false,"n":null,"t":true}'
+
+
+def test_empty_dict(sz):
+    assert sz.serialize({}, toBytes=False) == '{}'
+
+
+def test_top_level_bytes_base64(sz):
+    # The OrderedJsonEncoder.encode override base64-encodes a top-level
+    # bytes/bytearray value (b'raw' -> base64 'cmF3').
+    assert sz.serialize(b'raw', toBytes=False) == '"cmF3"'
+    assert sz.serialize(bytearray(b'raw'), toBytes=False) == '"cmF3"'
+
+
+def test_round_trip(sz):
+    data = {'name': 'Alice', 'n': 1, 'f': 1.5, 'b': True, 'list': [1, 'two', None]}
+    assert sz.deserialize(sz.serialize(data)) == data
+    assert sz.deserialize(sz.serialize(data, toBytes=False)) == data
+
+
+@pytest.mark.parametrize('value', [
+    {'z': b'raw'},      # bytes nested in a dict value
+    [b'raw'],           # bytes nested in a list
+    {'z': bytearray(b'raw')},
+])
+def test_nested_bytes_raise_typeerror(sz, value):
+    """
+    The bytes special-case in ``OrderedJsonEncoder.encode`` only fires for a
+    top-level value; bytes nested in a container raise TypeError. A fix, if
+    ever needed, belongs in ``OrderedJsonEncoder.default``.
+    """
+    with pytest.raises(TypeError):
+        sz.serialize(value)

plenum/common/channel.py

@@ -156,7 +156,7 @@ def _process_sync(self, msg: Any):
         # This is done so that messages can include additional metadata
         # isinstance is not used here because it returns true for NamedTuple
         # as well.
-        if not isinstance(type(msg), tuple):
+        if type(msg) is not tuple:
             msg = (msg,)
         handler = self._find_handler(msg[0])
         if handler is None:

plenum/common/util.py

@@ -342,6 +342,14 @@ def z85_to_friendly(z):
         return z
 
 
+def json_default_bytes_to_str(obj):
+    """``json.dumps`` default hook decoding bytes/bytearray as UTF-8."""
+    if isinstance(obj, (bytes, bytearray)):
+        return obj.decode()
+    raise TypeError('Object of type {} is not JSON '
+                    'serializable'.format(type(obj).__name__))
+
+
 def runWithLoop(loop, callback, *args, **kwargs):
     if loop.is_running():
         loop.call_soon(asyncio.ensure_future, callback(*args, **kwargs))

plenum/recorder/recorder.py

@@ -3,6 +3,7 @@
 import json
 from typing import Callable
 
+from plenum.common.util import json_default_bytes_to_str
 from storage.kv_store_rocksdb_int_keys import KeyValueStorageRocksdbIntKeys
 
 
@@ -46,7 +47,8 @@ def add_to_store(self, key, val):
             existing = json.loads(existing)
         except KeyError:
             existing = []
-        self.store.put(key, json.dumps([*existing, val]))
+        self.store.put(key, json.dumps([*existing, val],
+                                       default=json_default_bytes_to_str))
 
     def register_replay_target(self, id, target: Callable):
         assert id not in self.replay_targets

plenum/test/recorder/test_recorder.py

@@ -56,6 +56,19 @@ def test_add_to_recorder(recorder):
         i += 1
 
 
+def test_add_to_recorder_with_bytes(recorder):
+    msg1, frm1 = b'{"msg": "m1"}', b'f1'
+    msg2, to1 = b'{"msg": "m2"}', 't1'
+    recorder.add_incoming(msg1, frm1)
+    recorder.add_outgoing(msg2, to1)
+
+    all_msgs = []
+    for _, v in recorder.store.iterator(include_value=True):
+        all_msgs.extend(Recorder.get_parsed(v))
+    assert Recorder.filter_incoming(all_msgs) == [['{"msg": "m1"}', 'f1']]
+    assert Recorder.filter_outgoing(all_msgs) == [['{"msg": "m2"}', 't1']]
+
+
 def test_get_list_from_recorder(recorder):
     msg1, frm1 = 'm1', 'f1'
     msg2, frm2 = 'm2', 'f2'

scripts/test_zmq/test_zmq/zstack.py

@@ -556,7 +556,7 @@ def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
     @staticmethod
     def serializeMsg(msg):
         if isinstance(msg, Mapping):
-            msg = json.dumps(msg)
+            msg = json.dumps(msg, separators=(',', ':'))
         if isinstance(msg, str):
             msg = msg.encode()
         assert isinstance(msg, bytes)

stp_zmq/zstack.py

@@ -868,7 +868,7 @@ def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
     @staticmethod
     def serializeMsg(msg):
         if isinstance(msg, Mapping):
-            msg = json.dumps(msg)
+            msg = json.dumps(msg, separators=(',', ':'))
         if isinstance(msg, str):
             msg = msg.encode()
         assert isinstance(msg, bytes)