Fix regressions from ujson removal and add regression tests

Fixes layered on top of #1676's ujson removal. Dropping the dependency
resolves the four ujson CVEs affecting the previously pinned 1.33
(CVE-2022-31116, CVE-2022-31117, CVE-2026-44660, CVE-2026-54911;
CVE-2021-45958 cited in #1676 only affects ujson >= 1.34).

- channel.py: the lint fix in #1676 replaced `type(msg) != tuple` with
  `not isinstance(type(msg), tuple)`, which is always True and wrapped
  already-tuple messages into nested tuples, breaking handler routing.
  Restore the original semantics in a lint-clean form
  (`type(msg) is not tuple`), preserving NamedTuple wrapping.

- zstack serializeMsg: ujson emitted compact JSON; stdlib json's default
  separators add whitespace, growing every wire message and breaking the
  exact-size assertions in stp_zmq/test/test_zstack.py calibrated to
  MSG_LEN_LIMIT. Pass separators=(',', ':') to keep the wire format
  byte-compatible. (Applied to scripts/test_zmq's copy as well.)

- recorder: SimpleZStackWithRecorder records raw wire frames; ujson
  silently encoded bytes as UTF-8 strings while stdlib json raises
  TypeError, so recording mode (STACK_COMPANION=1) crashed on the first
  message. Add a bytes-decoding default hook and a bytes regression
  test.

- Add regression tests pinning JsonSerializer's exact byte output
  (key ordering, compact separators, raw-UTF-8 non-ASCII, float repr,
  int-key coercion, top-level-bytes base64) since it feeds
  signing/hashing, plus characterisation of the nested-bytes TypeError.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: kukgini <kukgini@gmail.com>

Author: kukgini

common/test/test_json_serializer_ujson_compat.py

@@ -0,0 +1,89 @@
+"""
+Regression tests pinning the exact byte output of ``JsonSerializer`` after the
+removal of the ``ujson`` dependency (CVE-2022-31116 / CVE-2022-31117 /
+CVE-2021-45958).
+
+``JsonSerializer`` feeds signing/hashing, so its output must stay byte-stable.
+ujson used to be the active encoder; these tests lock the stdlib-``json``
+``OrderedJsonEncoder`` output so any future drift (separators, key ordering,
+unicode escaping, float repr) is caught.
+
+The encoder is configured with ``sort_keys=True``, ``separators=(',', ':')``
+and ``ensure_ascii=False`` -- i.e. sorted keys, no whitespace, and raw UTF-8
+for non-ASCII characters (not ``\\uXXXX`` escapes).
+"""
+import pytest
+
+from common.serializers.json_serializer import JsonSerializer
+
+
+@pytest.fixture
+def sz():
+    return JsonSerializer()
+
+
+def test_keys_sorted_and_compact(sz):
+    # Insertion order must not affect output; keys are sorted, no whitespace.
+    assert sz.serialize({'b': 2, 'a': 1, 'c': 3}, toBytes=False) == '{"a":1,"b":2,"c":3}'
+
+
+def test_non_ascii_kept_raw_utf8(sz):
+    # ensure_ascii=False: characters are emitted as raw UTF-8, not \\uXXXX.
+    assert sz.serialize({'name': 'héllo', 'kr': '한글', 'emoji': '🚀'},
+                        toBytes=False) == '{"emoji":"🚀","kr":"한글","name":"héllo"}'
+    # And the bytes form is the UTF-8 encoding of that string.
+    assert sz.serialize({'name': 'héllo'}) == '{"name":"héllo"}'.encode('utf-8')
+
+
+def test_float_repr_pinned(sz):
+    assert sz.serialize({'a': 14.8639, 'b': -97.466179, 'c': 1.0, 'd': 1e20},
+                        toBytes=False) == '{"a":14.8639,"b":-97.466179,"c":1.0,"d":1e+20}'
+
+
+def test_int_keys_become_strings(sz):
+    # json coerces non-string keys to strings and then sorts lexicographically.
+    assert sz.serialize({3: 'c', 1: 'a', 2: 'b'}, toBytes=False) == '{"1":"a","2":"b","3":"c"}'
+
+
+def test_bool_and_none(sz):
+    assert sz.serialize({'t': True, 'f': False, 'n': None},
+                        toBytes=False) == '{"f":false,"n":null,"t":true}'
+
+
+def test_empty_dict(sz):
+    assert sz.serialize({}, toBytes=False) == '{}'
+
+
+def test_top_level_bytes_base64(sz):
+    # The OrderedJsonEncoder.encode override base64-encodes a top-level
+    # bytes/bytearray value (b'raw' -> base64 'cmF3').
+    assert sz.serialize(b'raw', toBytes=False) == '"cmF3"'
+    assert sz.serialize(bytearray(b'raw'), toBytes=False) == '"cmF3"'
+
+
+def test_round_trip(sz):
+    data = {'name': 'Alice', 'n': 1, 'f': 1.5, 'b': True, 'list': [1, 'two', None]}
+    assert sz.deserialize(sz.serialize(data)) == data
+    assert sz.deserialize(sz.serialize(data, toBytes=False)) == data
+
+
+@pytest.mark.parametrize('value', [
+    {'z': b'raw'},      # bytes nested in a dict value
+    [b'raw'],           # bytes nested in a list
+    {'z': bytearray(b'raw')},
+])
+def test_nested_bytes_raise_typeerror(sz, value):
+    """
+    KNOWN BEHAVIOURAL DIFFERENCE vs ujson.
+
+    The bytes special-case lives in ``OrderedJsonEncoder.encode`` and therefore
+    only fires for a *top-level* bytes value (see test above). With ujson
+    installed, bytes nested inside a container were silently encoded as UTF-8
+    strings; stdlib ``json`` raises ``TypeError`` instead.
+
+    This characterises the current behaviour so the difference is explicit and
+    tracked. If a future change needs nested raw bytes to round-trip, the fix
+    belongs in ``OrderedJsonEncoder.default`` (not here).
+    """
+    with pytest.raises(TypeError):
+        sz.serialize(value)

plenum/common/channel.py

@@ -156,7 +156,7 @@ def _process_sync(self, msg: Any):
         # This is done so that messages can include additional metadata
         # isinstance is not used here because it returns true for NamedTuple
         # as well.
-        if not isinstance(type(msg), tuple):
+        if type(msg) is not tuple:
             msg = (msg,)
         handler = self._find_handler(msg[0])
         if handler is None:

plenum/recorder/recorder.py

@@ -46,7 +46,18 @@ def add_to_store(self, key, val):
             existing = json.loads(existing)
         except KeyError:
             existing = []
-        self.store.put(key, json.dumps([*existing, val]))
+        self.store.put(key, json.dumps([*existing, val],
+                                       default=self._bytes_to_str))
+
+    @staticmethod
+    def _bytes_to_str(obj):
+        # Wire messages and ZMQ identities are recorded as raw bytes; ujson
+        # encoded them as UTF-8 strings, which stdlib json only does via
+        # this hook.
+        if isinstance(obj, (bytes, bytearray)):
+            return obj.decode()
+        raise TypeError('Object of type {} is not JSON '
+                        'serializable'.format(type(obj).__name__))
 
     def register_replay_target(self, id, target: Callable):
         assert id not in self.replay_targets

plenum/test/recorder/test_recorder.py

@@ -56,6 +56,22 @@ def test_add_to_recorder(recorder):
         i += 1
 
 
+def test_add_to_recorder_with_bytes(recorder):
+    # SimpleZStackWithRecorder passes the raw wire frames: both the message
+    # and the ZMQ identity arrive as undecoded bytes. ujson encoded bytes as
+    # UTF-8 strings; stdlib json must keep accepting them.
+    msg1, frm1 = b'{"msg": "m1"}', b'f1'
+    msg2, to1 = b'{"msg": "m2"}', 't1'
+    recorder.add_incoming(msg1, frm1)
+    recorder.add_outgoing(msg2, to1)
+
+    all_msgs = []
+    for _, v in recorder.store.iterator(include_value=True):
+        all_msgs.extend(Recorder.get_parsed(v))
+    assert Recorder.filter_incoming(all_msgs) == [['{"msg": "m1"}', 'f1']]
+    assert Recorder.filter_outgoing(all_msgs) == [['{"msg": "m2"}', 't1']]
+
+
 def test_get_list_from_recorder(recorder):
     msg1, frm1 = 'm1', 'f1'
     msg2, frm2 = 'm2', 'f2'

scripts/test_zmq/test_zmq/zstack.py

@@ -556,7 +556,9 @@ def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
     @staticmethod
     def serializeMsg(msg):
         if isinstance(msg, Mapping):
-            msg = json.dumps(msg)
+            # Compact form: message size limits assume no separator
+            # whitespace, as ujson emitted before it was removed.
+            msg = json.dumps(msg, separators=(',', ':'))
         if isinstance(msg, str):
             msg = msg.encode()
         assert isinstance(msg, bytes)

stp_zmq/zstack.py

@@ -868,7 +868,9 @@ def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
     @staticmethod
     def serializeMsg(msg):
         if isinstance(msg, Mapping):
-            msg = json.dumps(msg)
+            # Compact form: message size limits assume no separator
+            # whitespace, as ujson emitted before it was removed.
+            msg = json.dumps(msg, separators=(',', ':'))
         if isinstance(msg, str):
             msg = msg.encode()
         assert isinstance(msg, bytes)