Merge branch 'pylambda-cythonization-and-a-hypernewt' of github.com:hoytak/SFrame into pylambda-cythonization-and-a-hypernewt

Author: Hoyt Koepke

oss_src/unity/python/sframe/_scripts/CMakeLists.txt

@@ -4,14 +4,22 @@ set_source_files_properties("pyspark_unity.pyx" PROPERTIES CYTHON_IS_CXX TRUE)
 set_source_files_properties("../cython/cy_flexible_type.pyx" PROPERTIES CYTHON_IS_CXX TRUE)
 set_source_files_properties("../cython/cy_callback.pyx" PROPERTIES CYTHON_IS_CXX TRUE)
 
+# This is annoying
+get_directory_property( cur_include_directories INCLUDE_DIRECTORIES )
+list( APPEND cur_include_directories "${CMAKE_CURRENT_SOURCE_DIR}/../cython")
+set_directory_properties(PROPERTIES INCLUDE_DIRECTORIES "${cur_include_directories}")
+
 # To avoid complicated library dependencies on these modules, we need
 # to include additional cython files as part of this executable.
-cython_add_standalone_executable(pyspark_unity 
-  MAIN_MODULE "pyspark_unity.pyx" 
-  "../cython/cy_flexible_type.pyx" 
-  "../cython/cy_callback.pyx")
 
-set_property(TARGET pyspark_unity APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-rpath,$ORIGIN -Wl,-rpath,$ORIGIN/.. -Wl,-rpath,${CMAKE_CURRENT_BINARY_DIR} ")
+cython_add_standalone_executable(pyspark_unity MAIN_MODULE
+   "pyspark_unity.pyx" "../cython/cy_flexible_type.pyx"
+   "../cython/cy_callback.pyx")
+
+set_property(TARGET pyspark_unity
+             APPEND_STRING PROPERTY
+             LINK_FLAGS " -Wl,-rpath,$ORIGIN -Wl,-rpath,$ORIGIN/.. -Wl,-rpath,${CMAKE_CURRENT_BINARY_DIR} ")
+
 target_link_libraries(pyspark_unity python_callbacks spark_unity unity_core)
 add_dependencies(pyspark_unity release_binaries)
 

oss_src/unity/python/sframe/_scripts/pyspark_unity.pyx

@@ -3,10 +3,10 @@
 # in the CMakeLists pulls in the cython source files directly, so we
 # don't actually depend on these libraries at link time.
 
-from ..cython.cy_flexible_type cimport flexible_type, flex_type_enum, UNDEFINED
-from ..cython.cy_flexible_type cimport flexible_type_from_pyobject
-from ..cython.cy_flexible_type cimport pyobject_from_flexible_type
-from ..cython.cy_callback cimport register_exception
+from cy_flexible_type cimport flexible_type, flex_type_enum, UNDEFINED
+from cy_flexible_type cimport flexible_type_from_pyobject
+from cy_flexible_type cimport pyobject_from_flexible_type
+from cy_callback cimport register_exception
 
 from libcpp.string cimport string
 from libcpp.vector cimport vector
@@ -29,6 +29,10 @@ cdef extern from "<sframe/sframe.hpp>" namespace "graphlab":
         string column_name(size_t idx)
         size_t num_columns()
 
+cdef extern from "<cstdlib>":
+    void _exit(int)
+
+        
 cdef extern from "<sframe/sframe_iterators.hpp>" namespace "graphlab":
 
     cdef cppclass parallel_sframe_iterator_initializer:
@@ -106,7 +110,12 @@ cdef call_main():
     for i in range(len(args)):
         c_args[i] = args[i]
 
-    sys.exit(_spark_unity_main(len(args), c_args.data()))
+    # If this script is run with a different libpython minor version,
+    # e.g. 2.7.6 vs 2.7.10, or 2.7.11 vs 2.7.10, there appears to be
+    # race condition in tearing things down.  Everything appears to
+    # work besides that.  Using sys._exit instead of exit gets around
+    # this.
+    _exit(_spark_unity_main(len(args), c_args.data()))
 
 
 if __name__ == "__main__":

oss_src/unity/python/sframe/cython/cy_flexible_type.pyx

@@ -236,7 +236,8 @@ try:
     from ..data_structures import image
     _image_type = image.Image
     have_imagetype = True
-except ImportError:
+except Exception:  # relative import can raise ValueError or
+                   #ImportError, so catch anything
     have_imagetype = False
     _image_type = __bad_image