Merge branch 'master' into gl-pickle-extensibility

Conflicts:
	oss_src/unity/python/sframe/test/test_gl_pickler.py

Author: srikris

.travis.yml

@@ -1,6 +1,7 @@
 language: python
 python:
   - "2.7"
+  - "3.4"
 os:
   - linux
 

configure

@@ -4,7 +4,7 @@
 set -e
 RELEASE_DIR=release
 DEBUG_DIR=debug
-GRAPHLAB_HOME=$PWD
+GRAPHLAB_HOME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 DEPS_PREFIX=$PWD/deps/local
 
 if [[ $OSTYPE == linux* ]]; then
@@ -57,6 +57,8 @@ function print_help {
   echo
   echo "  --python_only           Only run python dependency installation."
   echo
+  echo "  --python3               Use Python 3.4, default is Python 2.7."
+  echo
   echo "  --toolchain=[toolchain] Toolchain is either a dato-deps file on disk, "
   echo "                          or a URL to a toolchain. If toolchain is"
   echo "                          \"default\", a default one is selected"
@@ -96,8 +98,8 @@ function unknown_option {
 function run_cleanup {
   #!/bin/bash
   echo "cleaning up";
-  rm -rf release debug deps cmake patches description.json
-  rm -f *.tar.gz 
+  rm -rf release debug deps cmake patches description.json deps_version dummy.cpp miniconda.sh
+  rm -f *.tar.gz
 } 
 
 
@@ -162,6 +164,7 @@ while [ $# -gt 0 ]
     --yes)                  default_yes=1;;
     --cmake_only)           cmake_only=1;;
     --python_only)          python_only=1;;
+    --python3)              python3=1;;
     --R_integration)        R_integration=1;;
     --help)                 print_help ;;
     -D)                     CFLAGS="$CFLAGS -D $2"; shift ;;
@@ -201,10 +204,20 @@ if [[ $python_only == 1 ]]; then
   run_cmake_configure=0
 fi
 
+if [[ $python3 == 1 ]]; then
+  CFLAGS="$CFLAGS -D PYTHON_VERSION=\"python3.4m\""
+  PYTHON_VERSION="python3.4m"
+else
+  CFLAGS="$CFLAGS -D PYTHON_VERSION=\"python2.7\""
+  PYTHON_VERSION="python2.7"
+fi
 
 if [[ $run_toolchain_install == 1 ]]; then
         run_cleanup
-        ./oss_local_scripts/install_python_toolchain.sh
+
+        cd ${GRAPHLAB_HOME}
+        PYTHON_VERSION=${PYTHON_VERSION} ./oss_local_scripts/install_python_toolchain.sh
+
         REMAINING_OPTIONS=$@
         if [[ $toolchain == default || $toolchain == "" ]]; then
                 if [[ $OSTYPE == darwin* ]]; then
@@ -251,13 +264,12 @@ if [[ $OSTYPE == msys ]]; then
         CXX=g++.exe
 fi
 CMAKE=$PWD/deps/local/bin/cmake
+
 LINKER=""
 if [ -e $PWD/deps/local/bin/ld.gold ]; then
         LINKER="-DCMAKE_LINKER=$PWD/deps/local/bin/ld.gold"
 elif [ -e $PWD/deps/local/bin/ld ]; then
         LINKER="-DCMAKE_LINKER=$PWD/deps/local/bin/ld"
-else
-        LINKER=""
 fi
 
 mkdir -p doc
@@ -292,9 +304,12 @@ if [[ $OSTYPE == msys ]]; then
   AR=`cygpath -m $AR`
 	CFLAGS="$CFLAGS -D CMAKE_AR=$AR"
         # a bunch of stuff needed for python
-        CFLAGS="$CFLAGS -D PYTHON_LIBRARY:FILEPATH=$PWD/deps/local/lib/python27.dll"
+        if [[ $python3 == 1 ]]; then
+            CFLAGS="$CFLAGS -D PYTHON_LIBRARY:FILEPATH=$PWD/deps/local/lib/python34.dll"
+        else
+            CFLAGS="$CFLAGS -D PYTHON_LIBRARY:FILEPATH=$PWD/deps/local/lib/python27.dll"
+        fi
         CFLAGS="$CFLAGS -D PYTHON_EXECUTABLE:FILEPATH=$PWD/deps/conda/bin/python"
-        CFLAGS="$CFLAGS -D PYTHON_INCLUDE_DIR:FILEPATH=$PWD/deps/local/include/python2.7"
         export PATH=$PATH:/mingw64/bin
 fi
 ## ============================================================================

local_cmake/UseCython.cmake

@@ -73,9 +73,9 @@ mark_as_advanced( CYTHON_ANNOTATE CYTHON_NO_DOCSTRINGS CYTHON_FLAGS )
 
 find_package( Cython REQUIRED )
 set(PYTHON_INCLUDE_DIR
-  ${CMAKE_SOURCE_DIR}/deps/local/include/python2.7)
+  ${CMAKE_SOURCE_DIR}/deps/local/include/${PYTHON_VERSION})
 set(PYTHON_LIBRARIES 
-  ${CMAKE_SOURCE_DIR}/deps/local/lib/${CMAKE_SHARED_LIBRARY_PREFIX}python2.7${CMAKE_SHARED_LIBRARY_SUFFIX})
+  ${CMAKE_SOURCE_DIR}/deps/local/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${PYTHON_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX})
 find_package( PythonLibs REQUIRED )
 
 
@@ -226,11 +226,16 @@ function( compile_pyx _name generated_file )
   list( REMOVE_DUPLICATES c_header_dependencies )
 
   # Add the command to run the compiler.
+  if (${PYTHON_VERSION} STREQUAL "python3.4m")
+    set( ADDITIONAL_CYTHON_PARAM, "-3")
+  else()
+    set( ADDITIONAL_CYTHON_PARAM, "")
+  endif()
   add_custom_command( OUTPUT ${_generated_file}
     COMMAND ${CYTHON_EXECUTABLE}
     ARGS ${cxx_arg} ${include_directory_arg}
     ${annotate_arg} ${no_docstrings_arg} ${cython_debug_arg} ${CYTHON_FLAGS} ${cython_traceback}
-    --output-file  ${_generated_target_file} ${pyx_locations} 
+    ${ADDITIONAL_CYTHON_PARAM} --output-file  ${_generated_target_file} ${pyx_locations}
     DEPENDS ${pyx_locations} ${pxd_dependencies}
     IMPLICIT_DEPENDS ${pyx_lang} ${c_header_dependencies}
     COMMENT ${comment}
@@ -261,7 +266,7 @@ function( cython_add_module _name )
   # add_custom_target(${_name}_headers DEPENDS ${generated_file})
   python_add_module( ${_name} ${generated_file} ${other_module_sources} )
   if(WIN32)
-          target_compile_definitions(${_name} PUBLIC MS_WIN64)
+    target_compile_definitions(${_name} PUBLIC MS_WIN64)
   endif()
   if( APPLE )
     set_target_properties( ${_name} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup" )

oss_local_scripts/conda_requirements.txt

@@ -18,4 +18,4 @@ six==1.9.0
 tornado==4.1
 wheel==0.24.0
 statsmodels
-PIL
+pillow

oss_local_scripts/install_python_toolchain.sh

@@ -28,8 +28,8 @@ fi
 function windows_patch_python_header {
 if [[ $OSTYPE == msys ]]; then
         echo "#include <math.h>" > tmp
-        cat deps/conda/include/python2.7/pyconfig.h >> tmp
-        cp tmp deps/conda/include/python2.7/pyconfig.h
+        cat deps/conda/include/${PYTHON_VERSION}/pyconfig.h >> tmp
+        cp tmp deps/conda/include/${PYTHON_VERSION}/pyconfig.h
         rm tmp
 fi
 }
@@ -81,7 +81,11 @@ if [[ $haspython == 0 ]]; then
                 cp $PWD/deps/conda/bin/*.dll $PWD/deps/conda/lib
         else
                 if [ ! -e miniconda.sh ]; then
-                        download_file http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh miniconda.sh
+                         if [[ ${PYTHON_VERSION} == "python3.4m" ]]; then
+                                  download_file http://repo.continuum.io/miniconda/Miniconda3-3.16.0-Linux-x86_64.sh miniconda.sh
+                         else
+                                  download_file http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh miniconda.sh
+                         fi
                 fi
                 bash ./miniconda.sh -p $PWD/deps/conda -b
         fi
@@ -90,8 +94,8 @@ $python_scripts/conda install -y --file oss_local_scripts/conda_requirements.txt
 $python_scripts/pip install -r oss_local_scripts/pip_requirements.txt
 # for windows
 if [ -e deps/conda/bin/include ]; then
-        mkdir -p deps/conda/include/python2.7
-        cp deps/conda/bin/include/* deps/conda/include/python2.7
+        mkdir -p deps/conda/include/${PYTHON_VERSION}
+        cp deps/conda/bin/include/* deps/conda/include/${PYTHON_VERSION}
 fi
 
 windows_patch_python_header
@@ -102,4 +106,4 @@ if [ $OSTYPE == "msys" ]; then
 else
   cp deps/conda/lib/libpython* deps/local/lib
 fi
-cp -R deps/conda/include/python2.7 deps/local/include
+cp -R deps/conda/include/${PYTHON_VERSION}/* deps/local/include

oss_local_scripts/make_egg.sh

@@ -93,9 +93,6 @@ source ${WORKSPACE}/oss_local_scripts/python_env.sh $build_type
 # Windows specific
 archive_file_ext="tar.gz"
 if [[ $OSTYPE == msys ]]; then
-  # C++ tests are default skipped on windows due to annoying issues around
-  # unable to find libstdc++.dll which are not so easily fixable
-  SKIP_CPP_TEST=1
   archive_file_ext="zip"
   unset PYTHONHOME
 fi
@@ -104,9 +101,17 @@ fi
 ### Build the source with version number ###
 build_source() {
   echo -e "\n\n\n================= Build ${BUILD_NUMBER} ================\n\n\n"
+
   # Configure
   cd ${WORKSPACE}
-  ./configure ${toolchain}
+
+  PY_MAJOR_VERSION=`python -V 2>&1 | perl -ne 'print m/^Python (\d)\.\d/'`
+  if [[ $PY_MAJOR_VERSION == 3 ]]; then
+      ./configure ${toolchain} --python3
+  else
+      ./configure ${toolchain}
+  fi
+
   # Make clean
   cd ${WORKSPACE}/${build_type}/oss_src/unity/python
   make oss_clean_python
@@ -115,8 +120,6 @@ build_source() {
   make -j${NUM_PROCS}
 
   if [[ -z $SKIP_CPP_TEST ]]; then
-      cd ${WORKSPACE}/oss_test
-      touch $(find . -name "*.cxx")
       cd ${WORKSPACE}/${build_type}/oss_test
       make -j${NUM_PROCS}
   fi
@@ -222,7 +225,7 @@ package_egg() {
   if [[ $OSTYPE == darwin* ]] || [[ $OSTYPE == msys ]]; then
     dist_type="bdist_wheel"
   fi
-  VERSION_NUMBER=`python -c "import sframe; print sframe.version"`
+  VERSION_NUMBER=`python -c "import sframe; print(sframe.version)"`
   ${PYTHON_EXECUTABLE} setup.py ${dist_type} # This produced an egg/wheel starting with SFrame-${VERSION_NUMBER} under dist/
 
   cd ${WORKSPACE}
@@ -236,7 +239,12 @@ package_egg() {
     mv ${EGG_PATH} ${NEW_EGG_PATH}
     EGG_PATH=${NEW_EGG_PATH}
   elif [[ $OSTYPE == msys ]]; then
-    EGG_PATH=${WORKSPACE}/${build_type}/oss_src/unity/python/dist/SFrame-${VERSION_NUMBER}-cp27-none-win_amd64.whl
+    EGG_PATH=`ls ${WORKSPACE}/${build_type}/oss_src/unity/python/dist/SFrame-${VERSION_NUMBER}-*-none-win_amd64.whl`
+  elif [[ $OSTYPE == linux-* ]]; then
+    PYTHON_VERSION=`$PYTHON_EXECUTABLE -V 2>&1 | perl -ne 'print m/^Python (\d\.\d)/'`
+    NEW_EGG_PATH=${WORKSPACE}/${build_type}/oss_src/unity/python/dist/SFrame-${VERSION_NUMBER}-py${PYTHON_VERSION}.${archive_file_ext}
+    mv ${EGG_PATH} ${NEW_EGG_PATH}
+    EGG_PATH=${NEW_EGG_PATH}
   fi
 
   # Install the egg and do a sanity check
@@ -276,6 +284,7 @@ if [[ -z $SKIP_BUILD ]]; then
   build_source
 fi
 
+set_build_number
 
 if [[ -z $SKIP_CPP_TEST ]]; then
   cpp_test 
@@ -285,12 +294,8 @@ if [[ -z $SKIP_TEST ]]; then
   unit_test
 fi
 
-set_build_number
-
 package_egg
 
 if [[ -z $SKIP_DOC ]]; then
   generate_docs
 fi
-
-

oss_local_scripts/pip_requirements.txt

@@ -1,10 +1,9 @@
 awscli==1.6.2
 httpretty==0.8.8
 librato-metrics==0.8.2
-mixpanel==4.0.2
+mixpanel==4.2.0
 moto==0.4.1
 prettytable==0.7.2
-wsgiref==0.1.2
 xmltodict==0.9.2
 argparse==1.2.1
 boto==2.33.0

oss_local_scripts/run_cpp_tests.py

@@ -24,7 +24,7 @@
   for root, dirnames, filenames in os.walk('.'):
     for filename in fnmatch.filter(filenames, '*.cxxtest*'):
       matches.append(os.path.join(root, filename))
-  print 'Found {} tests.'.format(len(matches))
+  print('Found {} tests.'.format(len(matches)))
 
   # Load the previous cache if it exists
   if os.path.exists(args.cache_file):
@@ -33,10 +33,10 @@
     cache = set()
 
   if type(cache) != set:
-    print "Invalid cache contents. Resetting cache"
+    print("Invalid cache contents. Resetting cache")
     cache = set()
 
-  print 'Found {} files in cache.'.format(len(cache))
+  print('Found {} files in cache.'.format(len(cache)))
 
   # Hash each test binary.
   new_tests = {}
@@ -48,18 +48,18 @@
       hasher.update(buf)
     new_tests[test_file] = hasher.hexdigest()
   elapsed = time.time() - start_time
-  print 'Hashed {0} files in {1} seconds.'.format(len(new_tests), elapsed)
+  print('Hashed {0} files in {1} seconds.'.format(len(new_tests), elapsed))
 
   # Make a list of tests whose hash does not appear in the cache
   tests = []
   for test_file in new_tests.keys():
       if new_tests[test_file] not in cache:
           tests.append(test_file)
-  print 'Ready to test {} files.'.format(len(tests))
+  print('Ready to test {} files.'.format(len(tests)))
 
   # If there are no tests, pick a random one.
   if len(tests) == 0:
-    print "For annoying reasons, we cannot handle running 0 tests because jenkins will complain. So we are running the first test"
+    print("For annoying reasons, we cannot handle running 0 tests because jenkins will complain. So we are running the first test")
     tests = [matches[0]]
 
   # Get basename and use .cxx rather than .cxxtest
@@ -68,16 +68,17 @@
   # Make the command to run
   cmd = 'ctest -j {0} --schedule-random -R "({1})"'.format(args.j, '|'.join(runtests))
   if args.dry_run:
-    print 'Dry run requested! Proposed ctest command:', cmd
+    print('Dry run requested! Proposed ctest command:', cmd)
     exit()
 
   exit_code = 0
   try: 
       ctest_output = subprocess.check_output(cmd, shell=True)
-  except subprocess.CalledProcessError, e:
+  except subprocess.CalledProcessError as e:
       ctest_output = e.output
       exit_code = e.returncode
-  print ctest_output
+  ctest_output = ctest_output.decode()
+  print(ctest_output)
   # get all the lines which say "Passed"
   lines = [line for line in ctest_output.splitlines() if "Passed" in line]
   # go through all the tests and see if we have a "Passed" line matching it

oss_local_scripts/run_python_test.sh

@@ -15,7 +15,6 @@ fi
 BUILD_TYPE=$1
 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 cd ${SCRIPT_DIR}/..
-oss_local_scripts/install_python_toolchain.sh
 source oss_local_scripts/python_env.sh $BUILD_TYPE
 
 # Make nosetest prints unity server log on exception

oss_src/lambda/graph_pylambda_master.cpp

@@ -33,15 +33,10 @@ void graph_pylambda_master::shutdown_instance() {
 } 
 
 graph_pylambda_master::graph_pylambda_master(size_t nworkers) {
-  std::vector<std::string> worker_addresses;
-  for (size_t i = 0; i < nworkers; ++i) {
-    worker_addresses.push_back(std::string("ipc://") + get_temp_name());
-  }
   m_worker_pool.reset(
       new worker_pool<graph_lambda_evaluator_proxy>(
           nworkers,
-          lambda_master::get_lambda_worker_binary(),
-          worker_addresses));
+          lambda_master::get_lambda_worker_binary()));
 
   if (nworkers < thread::cpu_count()) {
     logprogress_stream << "Using default " << nworkers << " lambda workers.\n";