Merge pull request #147 from esamanas/dbapi2-support-PRBRANCH

Add DBAPI2 support for SFrame

Author: Yucheng Low

oss_src/flexible_type/flexible_type_base_types.hpp

@@ -768,6 +768,27 @@ struct has_direct_conversion_to_flexible_type {
       flex_type_enum::UNDEFINED;
 };
 
+/**
+ * Given a set of types, choose a common type that all types in the set can be
+ * converted to and preserves the most data. Not designed to be passed a set
+ * with UNDEFINED in it.
+ */
+inline flex_type_enum get_common_type(const std::set<flex_type_enum> &types) {
+  if (types.size() == 0) return flex_type_enum::FLOAT;
+  else if (types.size() == 1) return *(types.begin());
+  else if (types.size() == 2) {
+    if (types.count(flex_type_enum::INTEGER) && types.count(flex_type_enum::FLOAT)) {
+      return flex_type_enum::FLOAT;
+    }
+    if (types.count(flex_type_enum::LIST) && types.count(flex_type_enum::VECTOR)) {
+      return flex_type_enum::LIST;
+    }
+  } else {
+    throw std::string("Could not find a common type to convert all values.");
+  }
+
+  return flex_type_enum::UNDEFINED;
+}
 
 } // namespace graphlab
 

oss_src/unity/lib/CMakeLists.txt

@@ -27,6 +27,8 @@ make_library(unity_core
     unity_odbc_connection.cpp
     image_util.cpp
     ../extensions/additional_sframe_utilities.cpp
+    unity_sarray_builder.cpp
+    unity_sframe_builder.cpp
   REQUIRES
     flexible_type
     pylambda table_printer

oss_src/unity/lib/api/unity_sarray_builder_interface.hpp

@@ -0,0 +1,30 @@
+/**
+ * Copyright (C) 2015 Dato, Inc.
+ * All rights reserved.
+ *
+ * This software may be modified and distributed under the terms
+ * of the BSD license. See the LICENSE file for details.
+ */
+#ifndef GRAPHLAB_UNITY_SARRAY_BUILDER_INTERFACE_HPP
+#define GRAPHLAB_UNITY_SARRAY_BUILDER_INTERFACE_HPP
+#include <vector>
+#include <flexible_type/flexible_type.hpp>
+#include <cppipc/magic_macros.hpp>
+
+namespace graphlab {
+
+class unity_sarray_base;
+
+GENERATE_INTERFACE_AND_PROXY(unity_sarray_builder_base, unity_sarray_builder_proxy,
+      (void, init, (size_t)(size_t)(flex_type_enum))
+      (void, append, (const flexible_type&)(size_t))
+      (void, append_multiple, (const std::vector<flexible_type>&)(size_t))
+      (flex_type_enum, get_type, )
+      (std::vector<flexible_type>, read_history, (size_t)(size_t))
+      (std::shared_ptr<unity_sarray_base>, close, )
+    )
+
+} // namespace graphlab
+
+#endif //GRAPHLAB_UNITY_SARRAY_BUILDER_INTERFACE_HPP
+#include <unity/lib/api/unity_sarray_interface.hpp>

oss_src/unity/lib/api/unity_sframe_builder_interface.hpp

@@ -0,0 +1,25 @@
+#ifndef GRAPHLAB_UNITY_SFRAME_BUILDER_INTERFACE_HPP
+#define GRAPHLAB_UNITY_SFRAME_BUILDER_INTERFACE_HPP
+#include <vector>
+#include <string>
+#include <flexible_type/flexible_type.hpp>
+#include <cppipc/magic_macros.hpp>
+
+namespace graphlab {
+
+class unity_sframe_base;
+
+GENERATE_INTERFACE_AND_PROXY(unity_sframe_builder_base, unity_sframe_builder_proxy,
+      (void, init, (size_t)(size_t)(std::vector<std::string>)(std::vector<flex_type_enum>)(std::string))
+      (void, append, (const std::vector<flexible_type>&)(size_t))
+      (void, append_multiple, (const std::vector<std::vector<flexible_type>>&)(size_t))
+      (std::vector<std::string>, column_names, )
+      (std::vector<flex_type_enum>, column_types, )
+      (std::vector<std::vector<flexible_type>>, read_history, (size_t)(size_t))
+      (std::shared_ptr<unity_sframe_base>, close, )
+    )
+
+} // namespace graphlab
+
+#endif //GRAPHLAB_UNITY_SFRAME_BUILDER_INTERFACE_HPP
+#include <unity/lib/api/unity_sframe_interface.hpp>

oss_src/unity/lib/gl_sarray.cpp

@@ -39,17 +39,12 @@ flex_type_enum infer_type_of_list(const std::vector<flexible_type>& vec) {
       last_type = val.get_type();
     }
   }
-  if (types.size() == 0) return flex_type_enum::FLOAT;
-  else if (types.size() == 1) return *(types.begin());
-  else if (types.size() == 2) {
-    if (types.count(flex_type_enum::INTEGER) && types.count(flex_type_enum::FLOAT)) {
-      return flex_type_enum::FLOAT;
-    }
-    if (types.count(flex_type_enum::LIST) && types.count(flex_type_enum::VECTOR)) {
-      return flex_type_enum::LIST;
-    }
+
+  try {
+    return get_common_type(types);
+  } catch(std::string &e) {
+    throw std::string("Cannot infer Array type. Not all elements of array are the same type.");
   }
-  throw std::string("Cannot infer Array type. Not all elements of array are the same type.");
 }
 
 /**

oss_src/unity/lib/unity_sarray_builder.cpp

@@ -0,0 +1,108 @@
+/**
+ * Copyright (C) 2015 Dato, Inc.
+ * All rights reserved.
+ *
+ * This software may be modified and distributed under the terms
+ * of the BSD license. See the LICENSE file for details.
+ */
+
+#include <unity/lib/unity_sarray_builder.hpp>
+#include <unity/lib/unity_sarray.hpp>
+
+namespace graphlab {
+
+void unity_sarray_builder::init(size_t num_segments, size_t history_size, flex_type_enum dtype) {
+  if(m_inited)
+    log_and_throw("This sarray_builder has already been initialized!");
+
+  m_sarray = std::make_shared<sarray<flexible_type>>();
+  m_sarray->open_for_write(num_segments);
+  m_out_iters.resize(num_segments);
+  m_history.resize(num_segments);
+  for(size_t i = 0; i < num_segments; ++i) {
+    m_out_iters[i] = m_sarray->get_output_iterator(i);
+    m_history[i] = std::make_shared<boost::circular_buffer<flexible_type>>(history_size);
+  }
+  m_given_dtype = dtype;
+  if(dtype == flex_type_enum::UNDEFINED)
+      log_and_throw("Must specify a valid type.");
+  m_sarray->set_type(m_given_dtype);
+
+  m_inited = true;
+}
+
+void unity_sarray_builder::append(const flexible_type &val, size_t segment) {
+  if(!m_inited)
+    log_and_throw("Must call 'init' first!");
+
+  if(m_closed)
+    log_and_throw("Cannot append values when closed.");
+
+  if(segment >= m_out_iters.size()) {
+    log_and_throw("Invalid segment number!");
+  }
+
+  m_history[segment]->push_back(val);
+
+  auto in_type = val.get_type();
+  if(in_type != flex_type_enum::UNDEFINED && in_type != m_given_dtype) {
+    log_and_throw(std::string("Append failed: ") +
+        flex_type_enum_to_name(in_type) + std::string(" type is "
+          "incompatible with type of future SArray."));
+  }
+
+  *(m_out_iters[segment]) = val;
+}
+
+void unity_sarray_builder::append_multiple(const std::vector<flexible_type> &vals, size_t segment) {
+  for(const auto &i : vals) {
+    this->append(i, segment);
+  }
+}
+
+flex_type_enum unity_sarray_builder::get_type() {
+  return m_given_dtype;
+}
+
+std::vector<flexible_type> unity_sarray_builder::read_history(size_t num_elems, size_t segment) {
+  if(!m_inited)
+    log_and_throw("Must call 'init' first!");
+
+  if(m_closed)
+    log_and_throw("History is invalid when closed.");
+
+  if(segment >= m_history.size())
+    log_and_throw("Invalid segment.");
+
+  auto history = m_history[segment];
+
+  if(num_elems > history->size())
+    num_elems = history->size();
+  if(num_elems == size_t(-1))
+    num_elems = history->size();
+
+  std::vector<flexible_type> ret_vec(num_elems);
+
+  if(num_elems == 0)
+    return ret_vec;
+
+  std::copy_n(history->rbegin(), num_elems, ret_vec.rbegin());
+
+  return ret_vec;
+}
+
+std::shared_ptr<unity_sarray_base> unity_sarray_builder::close() {
+  if(!m_inited)
+    log_and_throw("Must call 'init' first!");
+
+  if(m_closed)
+    log_and_throw("Already closed.");
+
+  m_sarray->close();
+  m_closed = true;
+  auto ret = std::make_shared<unity_sarray>();
+  ret->construct_from_sarray(m_sarray);
+  return ret;
+}
+
+} // namespace graphlab

oss_src/unity/lib/unity_sarray_builder.hpp

@@ -0,0 +1,108 @@
+/**
+ * Copyright (C) 2015 Dato, Inc.
+ * All rights reserved.
+ *
+ * This software may be modified and distributed under the terms
+ * of the BSD license. See the LICENSE file for details.
+ */
+#ifndef GRAPHLAB_UNITY_SARRAY_BUILDER_HPP
+#define GRAPHLAB_UNITY_SARRAY_BUILDER_HPP
+
+#include <vector>
+#include <sframe/sarray.hpp>
+#include <boost/circular_buffer.hpp>
+#include <unity/lib/api/unity_sarray_builder_interface.hpp>
+
+namespace graphlab {
+
+// forward declarations
+template <typename T>
+class sarray;
+
+/**
+ * Provides a Python interface to incrementally build an SArray.
+ *
+ * Unlike most other unity objects, this is not a wrapper of another
+ * "sarray_builder" class, but provides the implementation. This is because it
+ * is a slightly embellished wrapper around the SArray's output iterator, so
+ * there is no further functionality that needs to be available for the C++
+ * side. 
+ *
+ * The unity_sarray_builder is designed to append values until \ref close is
+ * called, which returns the SArray. No "reopening" is allowed, and no
+ * operations in that instance of unity_sarray_builder will work after close is
+ * called.
+ */
+class unity_sarray_builder: public unity_sarray_builder_base {
+ public:
+  /**
+   * Default constructor. Does nothing
+   */
+  unity_sarray_builder() {}
+
+  /**
+   * Initialize the unity_sarray_buidler.
+   *
+   * This essentially opens the output iterator for writing.
+   * 
+   */
+  void init(size_t num_segments, size_t history_size, flex_type_enum dtype);
+
+  /**
+   * Add a single flexible_type value to the SArray.
+   *
+   * The segment number allows the user to use the parallel interface provided
+   * by the underlying output_iterator.
+   *
+   * Throws if: 
+   *  - init hasn't been called or close has been called
+   *  - segment number is invalid
+   *  - the type of \p val differs from the type given in \ref init
+   *
+   */
+  void append(const flexible_type &val, size_t segment);
+
+  /**
+   * A wrapper around \ref append which adds multiple flexible_types to SArray.
+   *
+   * Throws if: 
+   *  - init hasn't been called or close has been called
+   *  - segment number is invalid
+   *  - the type of any values in \p vals differs from
+   *    the type given in \ref init
+   */
+  void append_multiple(const std::vector<flexible_type> &vals, size_t segment);
+
+  /**
+   * Return the current type of the SArray.
+   */
+  flex_type_enum get_type();
+
+  /**
+   * Return the last \p num_elems elements appended.
+   */
+  std::vector<flexible_type> read_history(size_t num_elems, size_t segment);
+
+  /**
+   * Finalize SArray and return it.
+   */
+  std::shared_ptr<unity_sarray_base> close();
+
+  unity_sarray_builder(const unity_sarray_builder&) = delete;
+  unity_sarray_builder& operator=(const unity_sarray_builder&) = delete;
+ private:
+  /// Methods
+
+  /// Variables
+  bool m_inited = false;
+  bool m_closed = false;
+  std::shared_ptr<sarray<flexible_type>> m_sarray;
+  std::vector<sarray<flexible_type>::iterator> m_out_iters;
+  flex_type_enum m_given_dtype = flex_type_enum::UNDEFINED;
+  std::set<flex_type_enum> m_types_inserted;
+
+  std::vector<std::shared_ptr<boost::circular_buffer<flexible_type>>> m_history;
+};
+
+} // namespace graphlab
+#endif // GRAPHLAB_UNITY_SARRAY_BUILDER_HPP