[FEA] Add Graph spectral clustering methods (#369)

Author: trxcllnt

modules/core/include/nv_node/utilities/napi_to_cpp.hpp

@@ -42,17 +42,23 @@ struct NapiToCPP {
     inline bool Has(Napi::Value key) const { return val.Has(key); }
     inline bool Has(const char* key) const { return val.Has(key); }
     inline bool Has(const std::string& key) const { return val.Has(key); }
-    inline NapiToCPP Get(napi_value key) const {
-      return Has(key) ? val.Get(key) : Env().Undefined();
+    inline NapiToCPP Get(napi_value key) const { return GetOrDefault(key, Env().Undefined()); }
+    inline NapiToCPP Get(Napi::Value key) const { return GetOrDefault(key, Env().Undefined()); }
+    inline NapiToCPP Get(const char* key) const { return GetOrDefault(key, Env().Undefined()); }
+    inline NapiToCPP Get(std::string const& key) const {
+      return GetOrDefault(key, Env().Undefined());
     }
-    inline NapiToCPP Get(Napi::Value key) const {
-      return Has(key) ? val.Get(key) : Env().Undefined();
+    inline NapiToCPP GetOrDefault(napi_value key, Napi::Value const& default_val) const {
+      return Has(key) ? val.Get(key) : default_val;
     }
-    inline NapiToCPP Get(const char* key) const {
-      return Has(key) ? val.Get(key) : Env().Undefined();
+    inline NapiToCPP GetOrDefault(Napi::Value key, Napi::Value const& default_val) const {
+      return Has(key) ? val.Get(key) : default_val;
     }
-    inline NapiToCPP Get(std::string const& key) const {
-      return Has(key) ? val.Get(key) : Env().Undefined();
+    inline NapiToCPP GetOrDefault(const char* key, Napi::Value const& default_val) const {
+      return Has(key) ? val.Get(key) : default_val;
+    }
+    inline NapiToCPP GetOrDefault(std::string const& key, Napi::Value const& default_val) const {
+      return Has(key) ? val.Get(key) : default_val;
     }
   };
 

modules/cudf/src/column.cpp

@@ -85,6 +85,7 @@ Napi::Function Column::Init(Napi::Env const& env, Napi::Object exports) {
                        // column/filling.cpp
                        StaticMethod<&Column::sequence>("sequence"),
                        // column/transform.cpp
+                       InstanceMethod<&Column::bools_to_mask>("boolsToMask"),
                        InstanceMethod<&Column::nans_to_nulls>("nansToNulls"),
                        // column/reduction.cpp
                        InstanceMethod<&Column::min>("min"),
@@ -211,6 +212,8 @@ Column::Column(CallbackArgs const& args) : EnvLocalObjectWrap<Column>(args) {
               : props.Has("nullMask")   ? props.Get("nullMask").As<Napi::Value>()
                                         : env.Null();
 
+  auto has_length = props.Has("length") && props.Get("length").IsNumber();
+
   switch (type().id()) {
     case cudf::type_id::INT8:
     case cudf::type_id::INT16:
@@ -229,7 +232,10 @@ Column::Column(CallbackArgs const& args) : EnvLocalObjectWrap<Column>(args) {
     case cudf::type_id::TIMESTAMP_MICROSECONDS:
     case cudf::type_id::TIMESTAMP_NANOSECONDS: {
       data_ = Napi::Persistent(data_to_devicebuffer(env, props.Get("data"), type()));
-      size_ = std::max(0, cudf::size_type(data_.Value()->size() / cudf::size_of(type())) - offset_);
+      size_ =
+        has_length
+          ? props.Get("length")
+          : std::max(0, cudf::size_type(data_.Value()->size() / cudf::size_of(type())) - offset_);
       null_mask_ =
         Napi::Persistent(mask.IsNull() ? data_to_null_bitmask(env, props.Get("data"), size_)
                                        : mask_to_null_bitmask(env, mask, size_));
@@ -245,7 +251,8 @@ Column::Column(CallbackArgs const& args) : EnvLocalObjectWrap<Column>(args) {
         }
       }(props.Get("children").As<Napi::Array>());
       data_      = Napi::Persistent(DeviceBuffer::New(env));
-      size_      = std::max(0, (num_children() > 0 ? child(0)->size() - 1 : 0) - offset_);
+      size_      = has_length ? props.Get("length")
+                              : std::max(0, (num_children() > 0 ? child(0)->size() - 1 : 0) - offset_);
       null_mask_ = Napi::Persistent(mask_to_null_bitmask(env, mask, size_));
       break;
     }
@@ -258,10 +265,11 @@ Column::Column(CallbackArgs const& args) : EnvLocalObjectWrap<Column>(args) {
       }(props.Get("children").As<Napi::Array>());
       data_ = Napi::Persistent(DeviceBuffer::New(env));
       if (num_children() > 0) {
-        size_ = std::max(0, child(0)->size() - offset_);
+        size_ = has_length ? props.Get("length") : std::max(0, child(0)->size() - offset_);
         for (cudf::size_type i = 0; ++i < num_children();) {
-          NODE_CUDF_EXPECT(
-            child(i)->size() == size_, "Struct column children must be the same size", env);
+          NODE_CUDF_EXPECT((child(i)->size() - offset_) == size_,
+                           "Struct column children must be the same size",
+                           env);
         }
       }
       null_mask_ = Napi::Persistent(mask_to_null_bitmask(env, mask, size_));
@@ -270,7 +278,7 @@ Column::Column(CallbackArgs const& args) : EnvLocalObjectWrap<Column>(args) {
     default: break;
   }
 
-  size_ = props.Has("length") ? props.Get("length") : size_;
+  // size_ = props.Has("length") ? props.Get("length") : size_;
 
   set_null_count([&]() -> cudf::size_type {
     if (!nullable()) { return 0; }

modules/cudf/src/column.ts

@@ -1249,13 +1249,20 @@ export interface Column<T extends DataType = any> {
    */
   dropNans(memoryResource?: MemoryResource): Column<T>;
 
+  /**
+   * Compress the data from a Bool8 Column to bits and return a Buffer
+   *
+   * @param memoryResource The optional MemoryResource used to allocate the result column's device
+   *   memory.
+   */
+  boolsToMask(memoryResource?: MemoryResource): [DeviceBuffer, number];
+
   /**
    * convert NaN values in the column with Null values,
    * while also updating the nullMask and nullCount values
    *
    * @param memoryResource The optional MemoryResource used to allocate the result column's device
    *   memory.
-   * @returns undefined if inplace=True, else updated column with Null values
    */
   nansToNulls(memoryResource?: MemoryResource): Column<T>;
 

modules/cudf/src/column/transform.cpp

@@ -13,19 +13,38 @@
 // limitations under the License.
 
 #include <node_cudf/column.hpp>
+
 #include <node_rmm/device_buffer.hpp>
 
-#include <napi.h>
 #include <cudf/column/column.hpp>
 #include <cudf/column/column_view.hpp>
 #include <cudf/null_mask.hpp>
 #include <cudf/table/table_view.hpp>
 #include <cudf/transform.hpp>
 #include <cudf/types.hpp>
+
 #include <rmm/device_buffer.hpp>
 
+#include <napi.h>
+
 namespace nv {
 
+std::pair<std::unique_ptr<rmm::device_buffer>, cudf::size_type> Column::bools_to_mask(
+  rmm::mr::device_memory_resource* mr) const {
+  try {
+    return cudf::bools_to_mask(*this, mr);
+  } catch (std::exception const& e) { NAPI_THROW(Napi::Error::New(Env(), e.what())); }
+}
+
+Napi::Value Column::bools_to_mask(Napi::CallbackInfo const& info) {
+  rmm::mr::device_memory_resource* mr = CallbackArgs{info}[0];
+  auto result                         = bools_to_mask(mr);
+  auto ary                            = Napi::Array::New(Env(), 2);
+  ary.Set(0u, DeviceBuffer::New(Env(), std::move(result.first)));
+  ary.Set(1u, result.second);
+  return ary;
+}
+
 std::pair<std::unique_ptr<rmm::device_buffer>, cudf::size_type> Column::nans_to_nulls(
   rmm::mr::device_memory_resource* mr) const {
   try {

modules/cudf/src/column_accessor.ts

@@ -61,7 +61,12 @@ export class ColumnAccessor<T extends TypeMap = any> {
 
   addColumns<R extends TypeMap>(data: ColumnsMap<R>|ColumnAccessor<R>) {
     data = (data instanceof ColumnAccessor) ? data._data : data;
-    return new ColumnAccessor({...this._data, ...data} as ColumnsMap<T&R>);
+    return new ColumnAccessor(
+      {...this._data, ...data} as ColumnsMap<{
+        [P in keyof(T & R)]: P extends keyof R ? R[P]                      //
+                                               : P extends keyof T ? T[P]  //
+                                                                   : never
+      }>);
   }
 
   dropColumns<R extends keyof T>(names: readonly R[]) {
@@ -70,7 +75,7 @@ export class ColumnAccessor<T extends TypeMap = any> {
     for (const name of this.names) {
       if (!(name in namesMap)) { data[name] = this._data[name]; }
     }
-    return new ColumnAccessor<Omit<T, R>>(data);
+    return new ColumnAccessor(data as ColumnsMap<{[P in Exclude<keyof T, R>]: T[P]}>);
   }
 
   selectByColumnName<R extends keyof T>(name: R) { return this.selectByColumnNames([name]); }

modules/cudf/src/data_frame.ts

@@ -312,7 +312,8 @@ export class DataFrame<T extends TypeMap = any> {
   /**
    * Return a new DataFrame with new columns added.
    *
-   * @param data mapping of names to new columns to add
+   * @param {SeriesMap<R>|DataFrame<R>} data mapping of names to new columns to add, or a GPU
+   *   DataFrame object
    *
    * @example
    * ```typescript
@@ -323,13 +324,6 @@ export class DataFrame<T extends TypeMap = any> {
    * df.assign({b: Series.new(["foo", "bar", "bar"])})
    * // returns df {a: [1, 2, 3], b: ["foo", "bar", "bar"]}
    * ```
-   */
-  assign<R extends TypeMap>(data: SeriesMap<R>): DataFrame<Omit<T, keyof R&string>&R>;
-
-  /**
-   * Return a new DataFrame with new columns added.
-   *
-   * @param data a GPU DataFrame object
    *
    * @example
    * ```typescript
@@ -341,8 +335,6 @@ export class DataFrame<T extends TypeMap = any> {
    * df.assign(df1) // returns df {a: [1, 2, 3], b: ["foo", "bar", "bar"]}
    * ```
    */
-  assign<R extends TypeMap>(data: DataFrame<R>): DataFrame<Omit<T, keyof R&string>&R>;
-
   assign<R extends TypeMap>(data: SeriesMap<R>|DataFrame<R>) {
     const columns = (data instanceof DataFrame) ? data._accessor : _seriesToColumns(data);
     return new DataFrame(this._accessor.addColumns(columns));

modules/cudf/src/node_cudf/column.hpp

@@ -686,10 +686,22 @@ struct Column : public EnvLocalObjectWrap<Column> {
     return sequence(env, size, init->operator cudf::scalar&(), step->operator cudf::scalar&(), mr);
   }
 
+  inline static Column::wrapper_t zeros(
+    Napi::Env const& env,
+    cudf::type_id type,
+    cudf::size_type size,
+    rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) {
+    auto zero = Scalar::New(env, Napi::Number::New(env, 0), cudf::data_type{type});
+    return sequence(env, size, zero->operator cudf::scalar&(), zero->operator cudf::scalar&(), mr);
+  }
+
   // column/transform.cpp
   std::pair<std::unique_ptr<rmm::device_buffer>, cudf::size_type> nans_to_nulls(
     rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) const;
 
+  std::pair<std::unique_ptr<rmm::device_buffer>, cudf::size_type> bools_to_mask(
+    rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) const;
+
   // column/copying.cpp
   Column::wrapper_t gather(
     Column const& gather_map,
@@ -909,6 +921,7 @@ struct Column : public EnvLocalObjectWrap<Column> {
   static Napi::Value sequence(Napi::CallbackInfo const& info);
 
   // column/transform.cpp
+  Napi::Value bools_to_mask(Napi::CallbackInfo const& info);
   Napi::Value nans_to_nulls(Napi::CallbackInfo const& info);
 
   // column/reductions.cpp

modules/cudf/src/series.ts

@@ -948,7 +948,7 @@ export class AbstractSeries<T extends DataType = any> {
    * For dictionary columns, the keys column component is copied and not trimmed if the gather
    * results in abandoned key elements.
    *
-   * @param selection A Series of 8/16/32-bit signed or unsigned integer indices to gather.
+   * @param indices A Series of 8/16/32-bit signed or unsigned integer indices to gather.
    * @param nullify_out_of_bounds If `true`, coerce rows that corresponds to out-of-bounds indices
    *   in the selection to null. If `false`, skips all bounds checking for selection values. Pass
    *   false if you are certain that the selection contains only valid indices for better
@@ -970,11 +970,11 @@ export class AbstractSeries<T extends DataType = any> {
    * c.gather(selection) // Bool8Series [true, true]
    * ```
    */
-  gather<R extends IndexType>(selection: Series<R>,
-                              nullify_out_of_bounds = false,
-                              memoryResource?: MemoryResource): Series<T> {
-    return this.__construct(
-      this._col.gather(selection._col, nullify_out_of_bounds, memoryResource));
+  gather(indices: Series<IndexType>|number[],
+         nullify_out_of_bounds = false,
+         memoryResource?: MemoryResource): Series<T> {
+    const map = Array.isArray(indices) ? Series.new(indices).cast(new Uint32) : indices;
+    return this.__construct(this._col.gather(map._col, nullify_out_of_bounds, memoryResource));
   }
 
   /**
@@ -1109,7 +1109,7 @@ export class AbstractSeries<T extends DataType = any> {
    * ```
    */
   scatter(value: T['scalarType'],
-          indices: Series<Int32>|number[],
+          indices: Series<IndexType>|number[],
           check_bounds?: boolean,
           memoryResource?: MemoryResource): Series<T>;
   /**
@@ -1135,24 +1135,24 @@ export class AbstractSeries<T extends DataType = any> {
    * ```
    */
   scatter(values: Series<T>,
-          indices: Series<Int32>|number[],
+          indices: Series<IndexType>|number[],
           check_bounds?: boolean,
           memoryResource?: MemoryResource): Series<T>;
 
   scatter(source: Series<T>|T['scalarType'],
-          indices: Series<Int32>|number[],
+          indices: Series<IndexType>|number[],
           check_bounds = false,
           memoryResource?: MemoryResource): Series<T> {
     const dst = new Table({columns: [this._col]});
-    const idx = Series.new(indices).cast(new Int32)._col;
+    const map = Array.isArray(indices) ? Series.new(indices).cast(new Uint32) : indices;
     if (source instanceof Series) {
       const src = new Table({columns: [source.cast(this.type)._col]});
       return this.__construct(
-        dst.scatterTable(src, idx, check_bounds, memoryResource).getColumnByIndex(0));
+        dst.scatterTable(src, map._col, check_bounds, memoryResource).getColumnByIndex(0));
     }
     const src = [new Scalar({type: this.type, value: source})];
     return this.__construct(
-      dst.scatterScalar(src, idx, check_bounds, memoryResource).getColumnByIndex(0));
+      dst.scatterScalar(src, map._col, check_bounds, memoryResource).getColumnByIndex(0));
   }
 
   /**
@@ -1668,7 +1668,6 @@ function asColumn<T extends DataType>(value: any) {
     if (Array.isArray(data)) {
       return fromArrow<T>(arrow.Vector.from({
         highWaterMark: Infinity,
-        nullValues: [undefined, null, NaN],
         type: value.type ?? inferType(data),
         // Slice `offset` from the Array before converting so
         // we don't write unnecessary values with the Arrow builders.
@@ -1677,12 +1676,14 @@ function asColumn<T extends DataType>(value: any) {
     }
 
     // If `data.buffer` is a ArrayBuffer, copy it to a DeviceBuffer
-    if (data.buffer instanceof ArrayBuffer) {
+    if (ArrayBuffer.isView(value) || (data.buffer instanceof ArrayBuffer)) {
+      if (typeof data.length === 'number') { value.length = data.length; }
       data   = new DeviceBuffer(typeof offset !== 'number' ? data : data.subarray(offset));
       offset = 0;
     }
     // If `data.buffer` is a DeviceBuffer, propagate its `byteOffset` to ColumnProps
     else if (data.buffer instanceof DeviceBuffer) {
+      if (typeof data.length === 'number') { value.length = data.length; }
       offset =
         (typeof offset !== 'number' ? 0 : offset) + (data.byteOffset / data.BYTES_PER_ELEMENT);
     }