Add msgpack timestamp support and fix C extension error handling

- Add timestamp=3 to msgpack.unpackb() to convert msgpack Timestamps
  to Python datetime objects
- Add datetime=True to msgpack.packb() to properly serialize datetime
  objects back to msgpack Timestamps
- Fix segfault in accel.c dump_rowdat_1 by adding NULL check after
  apply_transformer call
- Add unit tests for msgpack timestamp functionality
- Add UDF test functions for timestamp handling across numpy, pandas,
  polars, list, and non-vector types
- Fix .gitignore to only ignore test*.py at root level

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: kesmit13

.gitignore

@@ -83,7 +83,7 @@ dev-docs
 **/.ipynb_checkpoints
 **/.benchmarks
 *.ipynb
-test*.py
+/test*.py
 certs
 **/*.prof
 **/*.pprof

accel.c

@@ -4639,6 +4639,7 @@ static PyObject *dump_rowdat_1(PyObject *self, PyObject *args, PyObject *kwargs)
         while ((py_item = PyIter_Next(py_row_iter))) {
 
             py_item = apply_transformer(py_transformers[i], py_item);
+            if (!py_item) goto error;
             is_null = (uint8_t)(py_item == Py_None);
 
             CHECKMEM(1);

pyproject.toml

@@ -51,6 +51,7 @@ test = [
     "fastapi",
     "ipython",
     "jupysql",
+    "msgpack>=1.0.0",
     "pandas",
     "parameterized",
     "polars",

singlestoredb/functions/typing/__init__.py

@@ -12,6 +12,13 @@
 from typing import TypeVar
 from typing import Union
 
+try:
+    import msgpack
+    _has_msgpack = True
+except ImportError:
+    msgpack = None  # type: ignore[assignment]
+    _has_msgpack = False
+
 try:
     from typing import TypeVarTuple  # type: ignore
     from typing import Unpack  # type: ignore
@@ -153,10 +160,86 @@ def json_or_null_loads(v: Optional[str], **kwargs: Any) -> Optional[Any]:
 ]
 
 
+def msgpack_or_null_dumps(v: Optional[Any], **kwargs: Any) -> Optional[bytes]:
+    """
+    Serialize a Python object to MessagePack bytes or None.
+
+    Parameters
+    ----------
+    v : Optional[Any]
+        The Python object to serialize. If None or empty, the function returns None.
+    **kwargs : Any
+        Additional keyword arguments to pass to `msgpack.packb`.
+
+    Returns
+    -------
+    Optional[bytes]
+        The MessagePack bytes representation of the input object,
+        or None if the input is None or empty.
+
+    Raises
+    ------
+    ImportError
+        If msgpack is not installed.
+
+    """
+    if not _has_msgpack:
+        raise ImportError('msgpack is required for MessagePack serialization')
+    if not v:
+        return None
+    return msgpack.packb(v, datetime=True, **kwargs)
+
+
+# Force numpy dtype to 'object' to avoid issues with
+# numpy trying to infer the dtype and creating multidimensional arrays
+# instead of an array of Python objects.
+@output_type('object')
+def msgpack_or_null_loads(v: Optional[bytes], **kwargs: Any) -> Optional[Any]:
+    """
+    Deserialize MessagePack bytes to a Python object or None.
+
+    Parameters
+    ----------
+    v : Optional[bytes]
+        The MessagePack bytes to deserialize. If None or empty,
+        the function returns None.
+    **kwargs : Any
+        Additional keyword arguments to pass to `msgpack.unpackb`.
+
+    Returns
+    -------
+    Optional[Any]
+        The Python object represented by the MessagePack bytes,
+        or None if the input is None or empty.
+
+    Raises
+    ------
+    ImportError
+        If msgpack is not installed.
+
+    """
+    if not _has_msgpack:
+        raise ImportError('msgpack is required for MessagePack deserialization')
+    if not v:
+        return None
+    return msgpack.unpackb(v, raw=False, strict_map_key=False, timestamp=3, **kwargs)
+
+
+MessagePack: TypeAlias = Annotated[
+    Union[Dict[str, Any], List[Any], int, float, str, bool, bytes, None],
+    UDFAttrs(
+        sql_type=sql_types.BLOB(nullable=False),
+        args_transformer=msgpack_or_null_loads,
+        returns_transformer=msgpack_or_null_dumps,
+    ),
+]
+
+
 __all__ = [
     'Table',
     'Masked',
     'JSON',
+    'MessagePack',
     'UDFAttrs',
     'Transformer',
 ]

singlestoredb/functions/typing/numpy.py

@@ -11,6 +11,8 @@
 except ImportError:
     from typing_extensions import TypeAlias  # type: ignore
 
+from . import msgpack_or_null_dumps
+from . import msgpack_or_null_loads
 from . import UDFAttrs
 from . import json_or_null_dumps
 from . import json_or_null_loads
@@ -107,4 +109,30 @@ def default(self, obj: Any) -> Any:
 ]
 
 
-__all__ = ['array'] + [x for x in globals().keys() if x.endswith('Array')]
+def msgpack_numpy_default(obj: Any) -> Any:
+    """Default function for msgpack that handles numpy types."""
+    if isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
+    raise TypeError(f'Object of type {type(obj)} is not msgpack serializable')
+
+
+MessagePackArray: TypeAlias = Annotated[
+    npt.NDArray[np.object_],
+    UDFAttrs(
+        sql_type=sql_types.BLOB(nullable=False),
+        args_transformer=msgpack_or_null_loads,
+        returns_transformer=lambda x: msgpack_or_null_dumps(
+            x, default=msgpack_numpy_default,
+        ),
+    ),
+]
+
+
+__all__ = ['array'] + [
+    x for x in globals().keys()
+    if x.endswith('Array')
+]

singlestoredb/functions/typing/pandas.py

@@ -12,6 +12,8 @@
 except ImportError:
     from typing_extensions import TypeAlias  # type: ignore
 
+from . import msgpack_or_null_dumps
+from . import msgpack_or_null_loads
 from . import UDFAttrs
 from . import json_or_null_dumps
 from . import json_or_null_loads
@@ -109,4 +111,34 @@ def default(self, obj: Any) -> Any:
 ]
 
 
-__all__ = ['DataFrame'] + [x for x in globals().keys() if x.endswith('Series')]
+def msgpack_pandas_default(obj: Any) -> Any:
+    """Default function for msgpack that handles pandas types."""
+    if hasattr(obj, 'dtype') and hasattr(obj, 'tolist'):
+        # Handle pandas Series and numpy arrays
+        return obj.tolist()
+    elif isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif hasattr(obj, 'item'):
+        # Handle pandas scalar types
+        return obj.item()
+    raise TypeError(f'Object of type {type(obj)} is not msgpack serializable')
+
+
+MessagePackSeries: TypeAlias = Annotated[
+    pd.Series,
+    UDFAttrs(
+        sql_type=sql_types.BLOB(nullable=False),
+        args_transformer=msgpack_or_null_loads,
+        returns_transformer=lambda x: msgpack_or_null_dumps(
+            x, default=msgpack_pandas_default,
+        ),
+    ),
+]
+
+
+__all__ = ['DataFrame'] + [
+    x for x in globals().keys()
+    if x.endswith('Series')
+]

singlestoredb/functions/typing/polars.py

@@ -11,6 +11,8 @@
 except ImportError:
     from typing_extensions import TypeAlias  # type: ignore
 
+from . import msgpack_or_null_dumps
+from . import msgpack_or_null_loads
 from . import UDFAttrs
 from . import json_or_null_dumps
 from . import json_or_null_loads
@@ -115,4 +117,41 @@ def default(self, obj: Any) -> Any:
 ]
 
 
-__all__ = ['DataFrame'] + [x for x in globals().keys() if x.endswith('Series')]
+def msgpack_polars_default(obj: Any) -> Any:
+    """Default function for msgpack that handles polars types."""
+    if isinstance(obj, pl.Series):
+        # Convert Polars Series to Python list
+        return obj.to_list()
+    elif hasattr(obj, 'dtype') and \
+            str(obj.dtype).startswith(('Int', 'UInt', 'Float')):
+        # Handle Polars scalar integer and float types
+        return obj.item() if hasattr(obj, 'item') else obj
+    elif isinstance(
+        obj, (
+            pl.datatypes.Int8, pl.datatypes.Int16, pl.datatypes.Int32,
+            pl.datatypes.Int64, pl.datatypes.UInt8, pl.datatypes.UInt16,
+            pl.datatypes.UInt32, pl.datatypes.UInt64,
+        ),
+    ):
+        return int(obj)
+    elif isinstance(obj, (pl.datatypes.Float32, pl.datatypes.Float64)):
+        return float(obj)
+    raise TypeError(f'Object of type {type(obj)} is not msgpack serializable')
+
+
+MessagePackSeries: TypeAlias = Annotated[
+    pl.Series,
+    UDFAttrs(
+        sql_type=sql_types.BLOB(nullable=False),
+        args_transformer=msgpack_or_null_loads,
+        returns_transformer=lambda x: msgpack_or_null_dumps(
+            x, default=msgpack_polars_default,
+        ),
+    ),
+]
+
+
+__all__ = ['DataFrame'] + [
+    x for x in globals().keys()
+    if x.endswith('Series')
+]

singlestoredb/functions/typing/pyarrow.py

@@ -12,6 +12,8 @@
 except ImportError:
     from typing_extensions import TypeAlias  # type: ignore
 
+from . import msgpack_or_null_dumps
+from . import msgpack_or_null_loads  # noqa: F401
 from . import UDFAttrs
 from . import json_or_null_dumps
 from . import json_or_null_loads  # noqa: F401
@@ -113,4 +115,38 @@ def default(self, obj: Any) -> Any:
 ]
 
 
-__all__ = ['Table', 'array'] + [x for x in globals().keys() if x.endswith('Array')]
+def msgpack_pyarrow_default(obj: Any) -> Any:
+    """Default function for msgpack that handles pyarrow types."""
+    if hasattr(obj, 'as_py'):
+        # Handle PyArrow scalar types (including individual ints and floats)
+        return obj.as_py()
+    elif isinstance(obj, pa.Array):
+        # Convert PyArrow Array to Python list
+        return obj.to_pylist()
+    elif isinstance(obj, pa.Table):
+        # Convert PyArrow Table to list of dictionaries
+        return obj.to_pydict()
+    raise TypeError(f'Object of type {type(obj)} is not msgpack serializable')
+
+
+#
+# NOTE: We don't use input_transformer=msgpack_or_null_loads because it doesn't handle
+#       all cases (e.g., when the input is already a dict/list).
+#
+
+MessagePackArray: TypeAlias = Annotated[
+    pa.Array,
+    UDFAttrs(
+        sql_type=sql_types.BLOB(nullable=True),
+        # input_transformer=msgpack_or_null_loads,
+        returns_transformer=lambda x: msgpack_or_null_dumps(
+            x, default=msgpack_pyarrow_default,
+        ),
+    ),
+]
+
+
+__all__ = ['Table', 'array'] + [
+    x for x in globals().keys()
+    if x.endswith('Array')
+]