Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion native/core/benches/row_columnar.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
// under the License.

use arrow::datatypes::DataType as ArrowDataType;
use comet::execution::shuffle::row::{
use comet::execution::shuffle::spark_unsafe::row::{
process_sorted_row_partition, SparkUnsafeObject, SparkUnsafeRow,
};
use comet::execution::shuffle::CompressionCodec;
Expand Down
2 changes: 1 addition & 1 deletion native/core/src/execution/jni_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ use crate::{
errors::{try_unwrap_or_throw, CometError, CometResult},
execution::{
metrics::utils::update_comet_metric, planner::PhysicalPlanner, serde::to_arrow_datatype,
shuffle::row::process_sorted_row_partition, sort::RdxSort,
shuffle::spark_unsafe::row::process_sorted_row_partition, sort::RdxSort,
},
jvm_bridge::{jni_new_global_ref, JVMClasses},
};
Expand Down
4 changes: 1 addition & 3 deletions native/core/src/execution/shuffle/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@

pub(crate) mod codec;
mod comet_partitioning;
mod list;
mod map;
pub mod row;
pub mod spark_unsafe;
mod shuffle_writer;

pub use codec::{read_ipc_compressed, CompressionCodec, ShuffleBlockWriter};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

use crate::{
errors::CometError,
execution::shuffle::{
execution::shuffle::spark_unsafe::{
map::append_map_elements,
row::{append_field, downcast_builder_ref, SparkUnsafeObject, SparkUnsafeRow},
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

use crate::{
errors::CometError,
execution::shuffle::list::{append_to_builder, SparkUnsafeArray},
execution::shuffle::spark_unsafe::list::{append_to_builder, SparkUnsafeArray},
};
use arrow::array::builder::{ArrayBuilder, MapBuilder, MapFieldNames};
use arrow::datatypes::{DataType, FieldRef};
Expand Down
20 changes: 20 additions & 0 deletions native/core/src/execution/shuffle/spark_unsafe/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

mod list;
mod map;
pub mod row;
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ use crate::{
execution::{
shuffle::{
codec::{Checksum, ShuffleBlockWriter},
list::{append_list_element, SparkUnsafeArray},
map::{append_map_elements, get_map_key_value_fields, SparkUnsafeMap},
spark_unsafe::{
list::{append_list_element, SparkUnsafeArray},
map::{append_map_elements, get_map_key_value_fields, SparkUnsafeMap},
},
},
utils::bytes_to_i128,
},
Expand Down Expand Up @@ -293,7 +295,7 @@ pub(crate) use downcast_builder_ref;
/// `struct_builder.append` is called before/after calling this function to append the null buffer
/// of the struct array.
#[allow(clippy::redundant_closure_call)]
pub(crate) fn append_field(
pub(super) fn append_field(
dt: &DataType,
struct_builder: &mut StructBuilder,
row: &SparkUnsafeRow,
Expand Down Expand Up @@ -441,7 +443,7 @@ pub(crate) fn append_field(

/// Appends column of top rows to the given array builder.
#[allow(clippy::redundant_closure_call, clippy::too_many_arguments)]
pub(crate) fn append_columns(
fn append_columns(
row_addresses_ptr: *mut jlong,
row_sizes_ptr: *mut jint,
row_start: usize,
Expand Down
Loading