Skip to content

Commit 6a59b88

Browse files
SYaoJunSYaoJun
authored
chore: replace Buffer::copy_from with Buffer::from_iter for improved performance (#6749)
<!-- Thank you for submitting a pull request! We appreciate your time and effort. Please make sure to provide enough information so that we can review your pull request. The Summary and Testing sections below contain guidance on what to include. --> ## Summary Buffer::copy_from -> Buffer::from_iter <!-- If this PR is related to a tracked effort, please link to the relevant issue here (e.g., `Closes: #123`). Otherwise, feel free to ignore / delete this. In this section, please: 1. Explain the rationale for this change. 2. Summarize the changes included in this PR. A general rule of thumb is that larger PRs should have larger summaries. If there are a lot of changes, please help us review the code by explaining what was changed and why. If there is an issue or discussion attached, there is no need to duplicate all the details, but clarity is always preferred over brevity. --> Closes: #000 <!-- ## API Changes Uncomment this section if there are any user-facing changes. Consider whether the change affects users in one of the following ways: 1. Breaks public APIs in some way. 2. Changes the underlying behavior of one of the engine integrations. 3. Should some documentation be updated to reflect this change? If a public API is changed in a breaking manner, make sure to add the appropriate label. You can run `./scripts/public-api.sh` locally to see if there are any public API changes (and this also runs in our CI). --> ## Testing <!-- Please describe how this change was tested. Here are some common categories for testing in Vortex: 1. Verifying existing behavior is maintained. 2. Verifying new behavior and functionality works correctly. 3. Serialization compatibility (backwards and forwards) should be maintained or explicitly broken. --> --------- Signed-off-by: syaojun <libevent@yeah.net>
1 parent 9f06445 commit 6a59b88

File tree

6 files changed

+43
-95
lines changed

6 files changed

+43
-95
lines changed

‎encodings/fastlanes/src/delta/compute/cast.rs‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,16 +53,13 @@ mod tests {
5353
use vortex_array::dtype::DType;
5454
use vortex_array::dtype::Nullability;
5555
use vortex_array::dtype::PType;
56-
use vortex_buffer::Buffer;
56+
use vortex_buffer::buffer;
5757

5858
use crate::delta::DeltaArray;
5959

6060
#[test]
6161
fn test_cast_delta_u8_to_u32() {
62-
let primitive = PrimitiveArray::new(
63-
Buffer::copy_from(vec![10u8, 20, 30, 40, 50]),
64-
vortex_array::validity::Validity::NonNullable,
65-
);
62+
let primitive = PrimitiveArray::from_iter([10u8, 20, 30, 40, 50]);
6663
let array = DeltaArray::try_from_primitive_array(&primitive).unwrap();
6764

6865
let casted = array
@@ -83,7 +80,7 @@ mod tests {
8380
// DeltaArray doesn't support nullable arrays - the validity is handled at the DeltaArray level
8481
// Create a non-nullable array and then add validity to the DeltaArray
8582
let values = PrimitiveArray::new(
86-
Buffer::copy_from(vec![100u16, 0, 200, 300, 0]),
83+
buffer![100u16, 0, 200, 300, 0],
8784
vortex_array::validity::Validity::NonNullable,
8885
);
8986
let array = DeltaArray::try_from_primitive_array(&values).unwrap();
@@ -101,25 +98,25 @@ mod tests {
10198
#[rstest]
10299
#[case::u8(
103100
PrimitiveArray::new(
104-
Buffer::copy_from(vec![0u8, 10, 20, 30, 40, 50]),
101+
buffer![0u8, 10, 20, 30, 40, 50],
105102
vortex_array::validity::Validity::NonNullable,
106103
)
107104
)]
108105
#[case::u16(
109106
PrimitiveArray::new(
110-
Buffer::copy_from(vec![0u16, 100, 200, 300, 400, 500]),
107+
buffer![0u16, 100, 200, 300, 400, 500],
111108
vortex_array::validity::Validity::NonNullable,
112109
)
113110
)]
114111
#[case::u32(
115112
PrimitiveArray::new(
116-
Buffer::copy_from(vec![0u32, 1000, 2000, 3000, 4000]),
113+
buffer![0u32, 1000, 2000, 3000, 4000],
117114
vortex_array::validity::Validity::NonNullable,
118115
)
119116
)]
120117
#[case::u64(
121118
PrimitiveArray::new(
122-
Buffer::copy_from(vec![0u64, 10000, 20000, 30000]),
119+
buffer![0u64, 10000, 20000, 30000],
123120
vortex_array::validity::Validity::NonNullable,
124121
)
125122
)]

‎encodings/pco/src/array.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -573,15 +573,15 @@ mod tests {
573573
use vortex_array::arrays::PrimitiveArray;
574574
use vortex_array::assert_arrays_eq;
575575
use vortex_array::validity::Validity;
576-
use vortex_buffer::Buffer;
576+
use vortex_buffer::buffer;
577577

578578
use crate::PcoArray;
579579

580580
#[test]
581581
fn test_slice_nullable() {
582582
// Create a nullable array with some nulls
583583
let values = PrimitiveArray::new(
584-
Buffer::copy_from(vec![10u32, 20, 30, 40, 50, 60]),
584+
buffer![10u32, 20, 30, 40, 50, 60],
585585
Validity::from_iter([false, true, true, true, true, false]),
586586
);
587587
let pco = PcoArray::from_primitive(&values, 0, 128).unwrap();

‎encodings/pco/src/compute/cast.rs‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -59,16 +59,13 @@ mod tests {
5959
use vortex_array::dtype::Nullability;
6060
use vortex_array::dtype::PType;
6161
use vortex_array::validity::Validity;
62-
use vortex_buffer::Buffer;
62+
use vortex_buffer::buffer;
6363

6464
use crate::PcoArray;
6565

6666
#[test]
6767
fn test_cast_pco_f32_to_f64() {
68-
let values = PrimitiveArray::new(
69-
Buffer::copy_from(vec![1.0f32, 2.0, 3.0, 4.0, 5.0]),
70-
Validity::NonNullable,
71-
);
68+
let values = PrimitiveArray::from_iter([1.0f32, 2.0, 3.0, 4.0, 5.0]);
7269
let pco = PcoArray::from_primitive(&values, 0, 128).unwrap();
7370

7471
let casted = pco
@@ -89,10 +86,7 @@ mod tests {
8986
#[test]
9087
fn test_cast_pco_nullability_change() {
9188
// Test casting from NonNullable to Nullable
92-
let values = PrimitiveArray::new(
93-
Buffer::copy_from(vec![10u32, 20, 30, 40]),
94-
Validity::NonNullable,
95-
);
89+
let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]);
9690
let pco = PcoArray::from_primitive(&values, 0, 128).unwrap();
9791

9892
let casted = pco
@@ -101,14 +95,14 @@ mod tests {
10195
.unwrap();
10296
assert_arrays_eq!(
10397
casted,
104-
PrimitiveArray::new(Buffer::from_iter([10u32, 20, 30, 40]), Validity::AllValid,)
98+
PrimitiveArray::new(buffer![10u32, 20, 30, 40], Validity::AllValid,)
10599
);
106100
}
107101

108102
#[test]
109103
fn test_cast_sliced_pco_nullable_to_nonnullable() {
110104
let values = PrimitiveArray::new(
111-
Buffer::copy_from(vec![10u32, 20, 30, 40, 50, 60]),
105+
buffer![10u32, 20, 30, 40, 50, 60],
112106
Validity::from_iter([true, true, true, true, true, true]),
113107
);
114108
let pco = PcoArray::from_primitive(&values, 0, 128).unwrap();
@@ -148,23 +142,23 @@ mod tests {
148142

149143
#[rstest]
150144
#[case::f32(PrimitiveArray::new(
151-
Buffer::copy_from(vec![1.23f32, 4.56, 7.89, 10.11, 12.13]),
145+
buffer![1.23f32, 4.56, 7.89, 10.11, 12.13],
152146
Validity::NonNullable,
153147
))]
154148
#[case::f64(PrimitiveArray::new(
155-
Buffer::copy_from(vec![100.1f64, 200.2, 300.3, 400.4, 500.5]),
149+
buffer![100.1f64, 200.2, 300.3, 400.4, 500.5],
156150
Validity::NonNullable,
157151
))]
158152
#[case::i32(PrimitiveArray::new(
159-
Buffer::copy_from(vec![100i32, 200, 300, 400, 500]),
153+
buffer![100i32, 200, 300, 400, 500],
160154
Validity::NonNullable,
161155
))]
162156
#[case::u64(PrimitiveArray::new(
163-
Buffer::copy_from(vec![1000u64, 2000, 3000, 4000]),
157+
buffer![1000u64, 2000, 3000, 4000],
164158
Validity::NonNullable,
165159
))]
166160
#[case::single(PrimitiveArray::new(
167-
Buffer::copy_from(vec![42.42f64]),
161+
buffer![42.42f64],
168162
Validity::NonNullable,
169163
))]
170164
fn test_cast_pco_conformance(#[case] values: PrimitiveArray) {

‎encodings/pco/src/compute/mod.rs‎

Lines changed: 8 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,71 +8,46 @@ mod tests {
88
use rstest::rstest;
99
use vortex_array::arrays::PrimitiveArray;
1010
use vortex_array::compute::conformance::consistency::test_array_consistency;
11-
use vortex_buffer::Buffer;
1211

1312
use crate::PcoArray;
1413

1514
fn pco_f32() -> PcoArray {
16-
let values = PrimitiveArray::new(
17-
Buffer::copy_from(vec![1.23f32, 4.56, 7.89, 10.11, 12.13]),
18-
vortex_array::validity::Validity::NonNullable,
19-
);
15+
let values = PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]);
2016
PcoArray::from_primitive(&values, 0, 128).unwrap()
2117
}
2218

2319
fn pco_f64() -> PcoArray {
24-
let values = PrimitiveArray::new(
25-
Buffer::copy_from(vec![100.1f64, 200.2, 300.3, 400.4, 500.5]),
26-
vortex_array::validity::Validity::NonNullable,
27-
);
20+
let values = PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]);
2821
PcoArray::from_primitive(&values, 0, 128).unwrap()
2922
}
3023

3124
fn pco_i32() -> PcoArray {
32-
let values = PrimitiveArray::new(
33-
Buffer::copy_from(vec![100i32, 200, 300, 400, 500]),
34-
vortex_array::validity::Validity::NonNullable,
35-
);
25+
let values = PrimitiveArray::from_iter([100i32, 200, 300, 400, 500]);
3626
PcoArray::from_primitive(&values, 0, 128).unwrap()
3727
}
3828

3929
fn pco_u64() -> PcoArray {
40-
let values = PrimitiveArray::new(
41-
Buffer::copy_from(vec![1000u64, 2000, 3000, 4000]),
42-
vortex_array::validity::Validity::NonNullable,
43-
);
30+
let values = PrimitiveArray::from_iter([1000u64, 2000, 3000, 4000]);
4431
PcoArray::from_primitive(&values, 0, 128).unwrap()
4532
}
4633

4734
fn pco_i16() -> PcoArray {
48-
let values = PrimitiveArray::new(
49-
Buffer::copy_from(vec![10i16, 20, 30, 40, 50]),
50-
vortex_array::validity::Validity::NonNullable,
51-
);
35+
let values = PrimitiveArray::from_iter([10i16, 20, 30, 40, 50]);
5236
PcoArray::from_primitive(&values, 0, 128).unwrap()
5337
}
5438

5539
fn pco_i32_alt() -> PcoArray {
56-
let values = PrimitiveArray::new(
57-
Buffer::copy_from(vec![1i32, 2, 3, 4, 5]),
58-
vortex_array::validity::Validity::NonNullable,
59-
);
40+
let values = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]);
6041
PcoArray::from_primitive(&values, 0, 128).unwrap()
6142
}
6243

6344
fn pco_single() -> PcoArray {
64-
let values = PrimitiveArray::new(
65-
Buffer::copy_from(vec![42.42f64]),
66-
vortex_array::validity::Validity::NonNullable,
67-
);
45+
let values = PrimitiveArray::from_iter([42.42f64]);
6846
PcoArray::from_primitive(&values, 0, 128).unwrap()
6947
}
7048

7149
fn pco_large() -> PcoArray {
72-
let values = PrimitiveArray::new(
73-
Buffer::copy_from((0..1000).map(|i| i as u32).collect::<Vec<_>>()),
74-
vortex_array::validity::Validity::NonNullable,
75-
);
50+
let values = PrimitiveArray::from_iter(0u32..1000);
7651
PcoArray::from_primitive(&values, 3, 128).unwrap()
7752
}
7853

‎encodings/zstd/src/compute/cast.rs‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -82,16 +82,13 @@ mod tests {
8282
use vortex_array::dtype::Nullability;
8383
use vortex_array::dtype::PType;
8484
use vortex_array::validity::Validity;
85-
use vortex_buffer::Buffer;
85+
use vortex_buffer::buffer;
8686

8787
use crate::ZstdArray;
8888

8989
#[test]
9090
fn test_cast_zstd_i32_to_i64() {
91-
let values = PrimitiveArray::new(
92-
Buffer::copy_from(vec![1i32, 2, 3, 4, 5]),
93-
Validity::NonNullable,
94-
);
91+
let values = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]);
9592
let zstd = ZstdArray::from_primitive(&values, 0, 0).unwrap();
9693

9794
let casted = zstd
@@ -109,10 +106,7 @@ mod tests {
109106

110107
#[test]
111108
fn test_cast_zstd_nullability_change() {
112-
let values = PrimitiveArray::new(
113-
Buffer::copy_from(vec![10u32, 20, 30, 40]),
114-
Validity::NonNullable,
115-
);
109+
let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]);
116110
let zstd = ZstdArray::from_primitive(&values, 0, 0).unwrap();
117111

118112
let casted = zstd
@@ -128,7 +122,7 @@ mod tests {
128122
#[test]
129123
fn test_cast_sliced_zstd_nullable_to_nonnullable() {
130124
let values = PrimitiveArray::new(
131-
Buffer::copy_from(vec![10u32, 20, 30, 40, 50, 60]),
125+
buffer![10u32, 20, 30, 40, 50, 60],
132126
Validity::from_iter([true, true, true, true, true, true]),
133127
);
134128
let zstd = ZstdArray::from_primitive(&values, 0, 128).unwrap();
@@ -171,19 +165,19 @@ mod tests {
171165

172166
#[rstest]
173167
#[case::i32(PrimitiveArray::new(
174-
Buffer::copy_from(vec![100i32, 200, 300, 400, 500]),
168+
buffer![100i32, 200, 300, 400, 500],
175169
Validity::NonNullable,
176170
))]
177171
#[case::f64(PrimitiveArray::new(
178-
Buffer::copy_from(vec![1.1f64, 2.2, 3.3, 4.4, 5.5]),
172+
buffer![1.1f64, 2.2, 3.3, 4.4, 5.5],
179173
Validity::NonNullable,
180174
))]
181175
#[case::single(PrimitiveArray::new(
182-
Buffer::copy_from(vec![42i64]),
176+
buffer![42i64],
183177
Validity::NonNullable,
184178
))]
185179
#[case::large(PrimitiveArray::new(
186-
Buffer::copy_from((0..1000).map(|i| i as u32).collect::<Vec<_>>()),
180+
buffer![0u32..1000],
187181
Validity::NonNullable,
188182
))]
189183
fn test_cast_zstd_conformance(#[case] values: PrimitiveArray) {

‎encodings/zstd/src/compute/mod.rs‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -8,31 +8,22 @@ mod tests {
88
use rstest::rstest;
99
use vortex_array::arrays::PrimitiveArray;
1010
use vortex_array::compute::conformance::consistency::test_array_consistency;
11-
use vortex_buffer::Buffer;
11+
use vortex_buffer::buffer;
1212

1313
use crate::ZstdArray;
1414

1515
fn zstd_i32() -> ZstdArray {
16-
let values = PrimitiveArray::new(
17-
Buffer::copy_from(vec![100i32, 200, 300, 400, 500]),
18-
vortex_array::validity::Validity::NonNullable,
19-
);
16+
let values = PrimitiveArray::from_iter([100i32, 200, 300, 400, 500]);
2017
ZstdArray::from_primitive(&values, 0, 0).unwrap()
2118
}
2219

2320
fn zstd_f64() -> ZstdArray {
24-
let values = PrimitiveArray::new(
25-
Buffer::copy_from(vec![1.1f64, 2.2, 3.3, 4.4, 5.5]),
26-
vortex_array::validity::Validity::NonNullable,
27-
);
21+
let values = PrimitiveArray::from_iter([1.1f64, 2.2, 3.3, 4.4, 5.5]);
2822
ZstdArray::from_primitive(&values, 0, 0).unwrap()
2923
}
3024

3125
fn zstd_u32() -> ZstdArray {
32-
let values = PrimitiveArray::new(
33-
Buffer::copy_from(vec![10u32, 20, 30, 40, 50]),
34-
vortex_array::validity::Validity::NonNullable,
35-
);
26+
let values = PrimitiveArray::from_iter([10u32, 20, 30, 40, 50]);
3627
ZstdArray::from_primitive(&values, 0, 0).unwrap()
3728
}
3829

@@ -44,33 +35,30 @@ mod tests {
4435

4536
fn zstd_single() -> ZstdArray {
4637
let values = PrimitiveArray::new(
47-
Buffer::copy_from(vec![42i64]),
38+
buffer![42i64],
4839
vortex_array::validity::Validity::NonNullable,
4940
);
5041
ZstdArray::from_primitive(&values, 0, 0).unwrap()
5142
}
5243

5344
fn zstd_large() -> ZstdArray {
5445
let values = PrimitiveArray::new(
55-
Buffer::copy_from((0..1000).map(|i| i as u32).collect::<Vec<_>>()),
46+
buffer![0u32..1000],
5647
vortex_array::validity::Validity::NonNullable,
5748
);
5849
ZstdArray::from_primitive(&values, 3, 0).unwrap()
5950
}
6051

6152
fn zstd_all_same() -> ZstdArray {
6253
let values = PrimitiveArray::new(
63-
Buffer::copy_from(vec![42i32; 100]),
54+
buffer![42i32; 100],
6455
vortex_array::validity::Validity::NonNullable,
6556
);
6657
ZstdArray::from_primitive(&values, 0, 0).unwrap()
6758
}
6859

6960
fn zstd_negative() -> ZstdArray {
70-
let values = PrimitiveArray::new(
71-
Buffer::copy_from(vec![-100i32, -50, 0, 50, 100]),
72-
vortex_array::validity::Validity::NonNullable,
73-
);
61+
let values = PrimitiveArray::from_iter([-100i32, -50, 0, 50, 100]);
7462
ZstdArray::from_primitive(&values, 0, 0).unwrap()
7563
}
7664

0 commit comments

Comments
 (0)