[python] add Arrow serialization content_type

Author: Kobzol

Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update && \
     curl https://sh.rustup.rs -sSf | sh -s -- -y && \
     . $HOME/.cargo/env && \
     cargo install capnpc && \
-    pip3 install pycapnp cloudpickle pytest pytest-timeout cbor pytest-timeout && \
+    pip3 install pycapnp cloudpickle pytest pytest-timeout cbor pyarrow && \
     cargo build --all-features --release --verbose && \
     cd python && \
     python3 setup.py install

docs/guide/user.rst

@@ -180,6 +180,7 @@ Recognized content types:
   * cloudpickle - Serialized Python object via Cloudpickle
   * json - Object serialized into JSON
   * cbor - Object serialized into CBOR
+  * arrow - Object serialized with Apache Arrow
   * text - UTF-8 string.
   * text:<ENCODING> - Text with specified encoding
   * mime:<MIME> - Content type defined as MIME type

python/rain/common/content_type.py

@@ -4,7 +4,7 @@
 
 
 def check_content_type(name):
-    if name in [None, "pickle", "json", "dir", "text", "cbor",
+    if name in [None, "pickle", "json", "dir", "text", "cbor", "arrow",
                 "protobuf", "cloudpickle"]:
         return True
     if (name.startswith("text:") or
@@ -64,6 +64,9 @@ def encode_value(val, content_type):
     elif content_type == "cbor":
         import cbor
         d = cbor.dumps(val)
+    elif content_type == "arrow":
+        import pyarrow
+        d = pyarrow.serialize(val).to_buffer().to_pybytes()
     elif content_type.startswith("text"):
         if not isinstance(val, str):
             raise RainException("Encoding {!r} can only encode `str` objects."
@@ -102,6 +105,9 @@ def decode_value(data, content_type):
     elif content_type == "cbor":
         import cbor
         return cbor.loads(data)
+    elif content_type == "arrow":
+        import pyarrow
+        return pyarrow.deserialize(data)
     elif content_type.startswith("text"):
         if content_type == "text":
             enc = "utf-8"

python/requirements.txt

@@ -1,3 +1,4 @@
 cbor
 cloudpickle
+pyarrow
 pycapnp

tests/pytests/test_python.py

@@ -479,6 +479,7 @@ def test_gen(ctx, in1, in2, in3, in4, in5, *args, ina="bar", **kwargs):
         assert kwargs['kwA'] == ["A"]
         assert kwargs['kwB'] == ["B"]
         assert kwargs['kwC'] == ["C"]
+        assert kwargs['kwD'] == ["D"]
 
     @remote(inputs={'in1': Input(content_type='json'),
                     'in2': Input(content_type='pickle', load=True),
@@ -526,7 +527,9 @@ def copied(ctx, obj):
                 ina=blob("barbar", encode='cbor'),
                 kwA=pickled(["A"]),
                 kwB=blob(["B"], encode="json"),
-                kwC=blob(["C"], encode="cbor"))
+                kwC=blob(["C"], encode="cbor"),
+                kwD=blob(["D"], encode="arrow")
+            )
             s.submit()
             t1.wait()