[feat] support messages column as JSON string in iterable datasets (#147)

Add automatic JSON string decoding for the messages column in iterable
dataset loaders. This allows parquet files to store chat messages as
JSON strings instead of nested Arrow structs, avoiding schema inference
issues with deeply nested message formats.

Co-authored-by: mwxely <mwxely@users.noreply.github.com>

Author: mwxely

src/lmms_engine/datasets/iterable/qwen3_vl_iterable_dataset.py

@@ -1,3 +1,4 @@
+import json
 import os
 from typing import Dict, Tuple
 
@@ -22,6 +23,9 @@ def load_from_json(self, data, data_folder=None) -> Dict[str, torch.Tensor]:
         videos = []
         kwargs = {}
         messages = data["messages"]
+        # Support messages stored as JSON string (common in parquet files)
+        if isinstance(messages, str):
+            messages = json.loads(messages)
         for message in messages:
             for content in message["content"]:
                 if content["type"] == "image_url":

src/lmms_engine/datasets/iterable/vision_iterable_dataset.py

@@ -1,3 +1,4 @@
+import json
 import os
 from typing import Dict
 
@@ -63,6 +64,9 @@ def load_from_json(self, data, data_folder=None) -> Dict[str, torch.Tensor]:
         videos = []
         kwargs = {}
         messages = data["messages"]
+        # Support messages stored as JSON string (common in parquet files)
+        if isinstance(messages, str):
+            messages = json.loads(messages)
         for message in messages:
             for content in message["content"]:
                 if content["type"] == "image_url":
@@ -92,6 +96,8 @@ def load_from_json(self, data, data_folder=None) -> Dict[str, torch.Tensor]:
 
     def load_from_hf(self, data) -> Dict[str, torch.Tensor]:
         messages = data["messages"]
+        if isinstance(messages, str):
+            messages = json.loads(messages)
         hf_messages = TrainUtilities.convert_open_to_hf(messages)
         if isinstance(data["image"], list):
             images = data["image"]