Start SparkSQL backend for PySpark (new version number: 1.0.5)

Author: jpivarski

docs/conf.py

@@ -47,9 +47,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = "1.0.5-pre2"
+version = "1.0.5"
 # The full version, including alpha/beta/rc tags.
-release = "1.0.5-pre2"
+release = "1.0.5"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

histogrammar/defs.py

@@ -1090,6 +1090,9 @@ def _makeNPWeights(self, weights, shape):
         else:
             return weights * numpy.ones(shape, dtype=numpy.float64)
 
+    def fillsparksql(self, data):
+        pass
+
 # useful functions
 
 unweighted = named("unweighted", lambda datum: 1.0)

histogrammar/util.py

@@ -46,6 +46,7 @@ def __init__(self, container, fill):
         self.root = container.fillroot
         self.pycuda = container.fillpycuda
         self.numpy = container.fillnumpy
+        self.sparksql = container.fillsparksql
     def __call__(self, *args, **kwds):
         return self.fill(*args, **kwds)
 
@@ -212,8 +213,26 @@ def __init__(self, expr, name=None):
             self.name = expr.__name__
         else:
             self.name = name
-        if expr is not None and not isinstance(expr, (basestring, types.FunctionType)):
-            raise TypeError("quantity ({0}) must be a string or function".format(expr))
+        
+        if expr is None:
+            ok = True
+        elif isinstance(expr, basestring):
+            ok = True
+        elif isinstance(expr, types.FunctionType):
+            ok = True
+        else:
+            try:
+                from pyspark.sql.column import Column
+            except ImportError:
+                ok = False
+            else:
+                if isinstance(expr, Column):
+                    if self.name is None:
+                        self.name = str(expr)[7:-1]
+                ok = True
+        if not ok:
+            raise TypeError("quantity ({0}) must be a string, function, or SparkSQL Column".format(expr))
+
         if name is not None and not isinstance(name, basestring):
             raise TypeError("function name must be a string, not {0} (perhaps your arguments are reversed)".format(name))
 
@@ -287,6 +306,13 @@ def function(datum):
                 raise TypeError("immutable container (created from JSON or .ed) cannot be filled")
 
             else:
+                try:
+                    from pyspark.sql.column import Column
+                except ImportError:
+                    pass
+                else:
+                    if isinstance(self.expr, Column):
+                        raise TypeError("cannot use SparkSQL Column with the normal fill method; use fill.sparksql")
                 raise TypeError("unrecognized type for function: {0}".format(type(self.expr)))
 
         return self.fcn(*args, **kwds)

histogrammar/version.py

@@ -16,7 +16,7 @@
 
 import re
 
-__version__ = "1.0.5-pre2"
+__version__ = "1.0.5"
 
 version = __version__