Skip to content

Commit 0d2d550

Browse files
texodustexodus
committed
Add with_typed_array JavaScript APIs
Signed-off-by: Andrew Stein <steinlink@gmail.com>
1 parent 8c581e3 commit 0d2d550

18 files changed

Lines changed: 1206 additions & 35 deletions

File tree

‎Cargo.lock‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pnpm-lock.yaml‎

Lines changed: 41 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎rust/metadata/main.rs‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ use perspective_client::{
3535
ColumnWindow, DeleteOptions, JoinOptions, OnUpdateData, OnUpdateOptions, SystemInfo,
3636
TableInitOptions, UpdateOptions, ViewWindow,
3737
};
38+
use perspective_js::TypedArrayWindow;
3839
use perspective_viewer::config::{ViewerConfig, ViewerConfigUpdate};
3940
use ts_rs::TS;
4041

@@ -67,17 +68,18 @@ pub fn generate_type_bindings_js() -> Result<(), Box<dyn Error>> {
6768
let path = std::env::current_dir()?.join("../perspective-js/src/ts/ts-rs");
6869
ColumnType::export_all_to(&path)?;
6970
ColumnWindow::export_all_to(&path)?;
70-
ViewWindow::export_all_to(&path)?;
71-
TableInitOptions::export_all_to(&path)?;
72-
ViewConfigUpdate::export_all_to(&path)?;
71+
DeleteOptions::export_all_to(&path)?;
72+
JoinOptions::export_all_to(&path)?;
7373
OnUpdateData::export_all_to(&path)?;
7474
OnUpdateOptions::export_all_to(&path)?;
75-
JoinOptions::export_all_to(&path)?;
76-
UpdateOptions::export_all_to(&path)?;
77-
DeleteOptions::export_all_to(&path)?;
78-
ViewWindow::export_all_to(&path)?;
7975
SystemInfo::<f64>::export_all_to(&path)?;
76+
TableInitOptions::export_all_to(&path)?;
77+
TypedArrayWindow::export_all_to(&path)?;
78+
UpdateOptions::export_all_to(&path)?;
8079
ViewConfig::export_all_to(&path)?;
80+
ViewConfigUpdate::export_all_to(&path)?;
81+
ViewWindow::export_all_to(&path)?;
82+
ViewWindow::export_all_to(&path)?;
8183
Ok(())
8284
}
8385

‎rust/perspective-client/perspective.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ message ViewPort {
8686
optional uint32 start_col = 2;
8787
optional uint32 end_row = 3;
8888
optional uint32 end_col = 4;
89+
optional bool emit_legacy_row_path_names = 5;
8990
// optional bool id = 5;
9091
// optional bool index = 3;
9192
// optional bool formatted = 6;

‎rust/perspective-client/src/rust/view.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,13 @@ pub struct ViewWindow {
109109
#[ts(optional)]
110110
#[serde(skip_serializing_if = "Option::is_none")]
111111
pub compression: Option<String>,
112+
113+
/// When `true`, group-by columns use legacy `"colname (Group by N)"`
114+
/// naming. When `false`, they use `__ROW_PATH_N__` naming consistent
115+
/// with the SQL backend. Defaults to `true` for backwards compatibility.
116+
#[ts(optional)]
117+
#[serde(skip_serializing_if = "Option::is_none")]
118+
pub emit_legacy_row_path_names: Option<bool>,
112119
}
113120

114121
impl From<ViewWindow> for ViewPort {
@@ -118,6 +125,7 @@ impl From<ViewWindow> for ViewPort {
118125
start_col: window.start_col.map(|x| x.floor() as u32),
119126
end_row: window.end_row.map(|x| x.ceil() as u32),
120127
end_col: window.end_col.map(|x| x.ceil() as u32),
128+
emit_legacy_row_path_names: window.emit_legacy_row_path_names,
121129
}
122130
}
123131
}
@@ -129,6 +137,7 @@ impl From<ViewPort> for ViewWindow {
129137
start_col: window.start_col.map(|x| x as f64),
130138
end_row: window.end_row.map(|x| x as f64),
131139
end_col: window.end_col.map(|x| x as f64),
140+
emit_legacy_row_path_names: window.emit_legacy_row_path_names,
132141
..ViewWindow::default()
133142
}
134143
}

‎rust/perspective-client/src/rust/virtual_server/data.rs‎

Lines changed: 75 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,15 @@ use std::error::Error;
1414
use std::sync::Arc;
1515

1616
use arrow_array::builder::{
17-
BooleanBuilder, Float64Builder, Int32Builder, StringBuilder, TimestampMillisecondBuilder,
17+
BooleanBuilder, Float64Builder, Int32Builder, StringDictionaryBuilder,
18+
TimestampMillisecondBuilder,
1819
};
20+
use arrow_array::cast::AsArray;
21+
use arrow_array::types::Int32Type;
1922
use arrow_array::{
20-
Array, ArrayRef, BooleanArray, Date32Array, Date64Array, Decimal128Array, Float32Array,
21-
Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, LargeStringArray, RecordBatch,
22-
StringArray, Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray,
23+
Array, ArrayAccessor, ArrayRef, BooleanArray, Date32Array, Date64Array, Decimal128Array,
24+
Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, LargeStringArray,
25+
RecordBatch, StringArray, Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray,
2326
Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray,
2427
TimestampNanosecondArray, TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array,
2528
UInt64Array,
@@ -36,12 +39,16 @@ use crate::config::{GroupRollupMode, Scalar, ViewConfig};
3639
/// [`VirtualDataSlice`].
3740
pub enum ColumnBuilder {
3841
Boolean(BooleanBuilder),
39-
String(StringBuilder),
42+
String(StringDictionaryBuilder<Int32Type>),
4043
Float(Float64Builder),
4144
Integer(Int32Builder),
4245
Datetime(TimestampMillisecondBuilder),
4346
}
4447

48+
fn dict_data_type() -> DataType {
49+
DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
50+
}
51+
4552
/// A single cell value in a row-oriented data representation.
4653
///
4754
/// Used when converting [`VirtualDataSlice`] to row format for JSON
@@ -90,7 +97,7 @@ impl SetVirtualDataColumn for Option<String> {
9097
}
9198

9299
fn new_builder() -> ColumnBuilder {
93-
ColumnBuilder::String(StringBuilder::new())
100+
ColumnBuilder::String(StringDictionaryBuilder::new())
94101
}
95102

96103
fn to_scalar(self) -> Scalar {
@@ -277,6 +284,11 @@ fn extract_scalar(array: &ArrayRef, row_idx: usize) -> Scalar {
277284
let arr = array.as_any().downcast_ref::<StringArray>().unwrap();
278285
Scalar::String(arr.value(row_idx).to_string())
279286
},
287+
DataType::Dictionary(..) => {
288+
let dict = array.as_dictionary::<Int32Type>();
289+
let values = dict.downcast_dict::<StringArray>().unwrap();
290+
Scalar::String(values.value(row_idx).to_string())
291+
},
280292
DataType::Float64 => {
281293
let arr = array.as_any().downcast_ref::<Float64Array>().unwrap();
282294
Scalar::Float(arr.value(row_idx))
@@ -350,14 +362,26 @@ fn coerce_column(
350362
array: &ArrayRef,
351363
) -> Result<(Field, ArrayRef), Box<dyn Error>> {
352364
match field.data_type() {
353-
DataType::Boolean
354-
| DataType::Utf8
355-
| DataType::Float64
356-
| DataType::Int32
357-
| DataType::Date32 => Ok((
365+
DataType::Boolean | DataType::Float64 | DataType::Int32 | DataType::Date32 => Ok((
358366
Field::new(name, field.data_type().clone(), true),
359367
array.clone(),
360368
)),
369+
DataType::Dictionary(..) => Ok((Field::new(name, dict_data_type(), true), array.clone())),
370+
DataType::Utf8 => {
371+
let arr = array.as_any().downcast_ref::<StringArray>().unwrap();
372+
let mut builder = StringDictionaryBuilder::<Int32Type>::new();
373+
for i in 0..arr.len() {
374+
if arr.is_null(i) {
375+
builder.append_null();
376+
} else {
377+
builder.append_value(arr.value(i));
378+
}
379+
}
380+
Ok((
381+
Field::new(name, dict_data_type(), true),
382+
Arc::new(builder.finish()) as ArrayRef,
383+
))
384+
},
361385
DataType::Timestamp(TimeUnit::Millisecond, _) => Ok((
362386
Field::new(name, DataType::Timestamp(TimeUnit::Millisecond, None), true),
363387
array.clone(),
@@ -502,20 +526,27 @@ fn coerce_column(
502526
},
503527
DataType::LargeUtf8 => {
504528
let arr = array.as_any().downcast_ref::<LargeStringArray>().unwrap();
505-
let result: StringArray = arr.iter().map(|v| v.map(|v| v.to_string())).collect();
529+
let mut builder = StringDictionaryBuilder::<Int32Type>::new();
530+
for i in 0..arr.len() {
531+
if arr.is_null(i) {
532+
builder.append_null();
533+
} else {
534+
builder.append_value(arr.value(i));
535+
}
536+
}
506537
Ok((
507-
Field::new(name, DataType::Utf8, true),
508-
Arc::new(result) as ArrayRef,
538+
Field::new(name, dict_data_type(), true),
539+
Arc::new(builder.finish()) as ArrayRef,
509540
))
510541
},
511542
dt => {
512543
tracing::warn!(
513-
"Coercing unknown Arrow type {} to Utf8 for column '{}'",
544+
"Coercing unknown Arrow type {} to Dictionary for column '{}'",
514545
dt,
515546
name
516547
);
517548
let num_rows = array.len();
518-
let mut builder = StringBuilder::new();
549+
let mut builder = StringDictionaryBuilder::<Int32Type>::new();
519550
for i in 0..num_rows {
520551
if array.is_null(i) {
521552
builder.append_null();
@@ -525,7 +556,7 @@ fn coerce_column(
525556
}
526557
}
527558
Ok((
528-
Field::new(name, DataType::Utf8, true),
559+
Field::new(name, dict_data_type(), true),
529560
Arc::new(builder.finish()) as ArrayRef,
530561
))
531562
},
@@ -667,9 +698,10 @@ impl VirtualDataSlice {
667698
Field::new(name, DataType::Boolean, true),
668699
Arc::new(b.finish()),
669700
),
670-
ColumnBuilder::String(b) => {
671-
(Field::new(name, DataType::Utf8, true), Arc::new(b.finish()))
672-
},
701+
ColumnBuilder::String(b) => (
702+
Field::new(name, dict_data_type(), true),
703+
Arc::new(b.finish()),
704+
),
673705
ColumnBuilder::Float(b) => (
674706
Field::new(name, DataType::Float64, true),
675707
Arc::new(b.finish()),
@@ -736,7 +768,9 @@ impl VirtualDataSlice {
736768
let cell = if col.is_null(row_idx) {
737769
match field.data_type() {
738770
DataType::Boolean => VirtualDataCell::Boolean(None),
739-
DataType::Utf8 => VirtualDataCell::String(None),
771+
DataType::Utf8 | DataType::Dictionary(..) => {
772+
VirtualDataCell::String(None)
773+
},
740774
DataType::Float64 => VirtualDataCell::Float(None),
741775
DataType::Int32 => VirtualDataCell::Integer(None),
742776
DataType::Timestamp(TimeUnit::Millisecond, _) => {
@@ -754,6 +788,11 @@ impl VirtualDataSlice {
754788
let arr = col.as_any().downcast_ref::<StringArray>().unwrap();
755789
VirtualDataCell::String(Some(arr.value(row_idx).to_string()))
756790
},
791+
DataType::Dictionary(..) => {
792+
let dict = col.as_dictionary::<Int32Type>();
793+
let values = dict.downcast_dict::<StringArray>().unwrap();
794+
VirtualDataCell::String(Some(values.value(row_idx).to_string()))
795+
},
757796
DataType::Float64 => {
758797
let arr = col.as_any().downcast_ref::<Float64Array>().unwrap();
759798
VirtualDataCell::Float(Some(arr.value(row_idx)))
@@ -832,6 +871,21 @@ impl VirtualDataSlice {
832871
.collect::<Vec<_>>(),
833872
)?
834873
},
874+
DataType::Dictionary(..) => {
875+
let dict = col.as_dictionary::<Int32Type>();
876+
let values = dict.downcast_dict::<StringArray>().unwrap();
877+
serde_json::to_value(
878+
(0..num_rows)
879+
.map(|i| {
880+
if col.is_null(i) {
881+
None
882+
} else {
883+
Some(values.value(i))
884+
}
885+
})
886+
.collect::<Vec<_>>(),
887+
)?
888+
},
835889
DataType::Float64 => {
836890
let arr = col.as_any().downcast_ref::<Float64Array>().unwrap();
837891
serde_json::to_value(

‎rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,7 @@ fn test_view_get_data_col_sort_ascending() {
311311
end_row: Some(100),
312312
start_col: Some(0),
313313
end_col: None,
314+
..ViewPort::default()
314315
};
315316

316317
let mut schema = IndexMap::new();
@@ -341,6 +342,7 @@ fn test_view_get_data_col_sort_descending() {
341342
end_row: Some(100),
342343
start_col: Some(0),
343344
end_col: None,
345+
..ViewPort::default()
344346
};
345347

346348
let mut schema = IndexMap::new();
@@ -370,6 +372,7 @@ fn test_view_get_data() {
370372
end_row: Some(100),
371373
start_col: Some(0),
372374
end_col: Some(5),
375+
..ViewPort::default()
373376
};
374377

375378
let mut schema = IndexMap::new();
@@ -536,6 +539,7 @@ fn test_view_get_data_flat_no_grouping_id() {
536539
end_row: Some(100),
537540
start_col: Some(0),
538541
end_col: None,
542+
..ViewPort::default()
539543
};
540544

541545
let mut schema = IndexMap::new();

‎rust/perspective-js/Cargo.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@ wasm-bindgen-test = "0.3.13"
6464

6565
[dependencies]
6666
perspective-client = { version = "4.4.1", features = ["sendable"] }
67+
arrow-array = { version = "57.3.0", default-features = false }
68+
arrow-ipc = { version = "57.3.0", default-features = false }
69+
arrow-schema = { version = "57.3.0", default-features = false }
6770
bytes = "1.10.1"
6871
chrono = "0.4"
6972
derivative = "2.2.0"

0 commit comments

Comments
 (0)