Skip to content

Commit 907639b

Browse files
zxch3nzxch3n
authored
feat(rs): add utf16 api for LoroText (#871)
* feat(rs): add utf16 api for LoroText * feat: add a new method to convert between utf8/16/unicode
1 parent 6cfef7f commit 907639b

26 files changed

Lines changed: 765 additions & 361 deletions

‎Cargo.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/loro-internal/benches/encode.rs‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ mod sync {
44

55
use super::*;
66
use bench_utils::{get_automerge_actions, TextAction};
7-
use loro_internal::{encoding::ExportMode, LoroDoc};
7+
use loro_internal::{cursor::PosType, encoding::ExportMode, LoroDoc};
88

99
pub fn b4(c: &mut Criterion) {
1010
let actions = get_automerge_actions();
@@ -25,16 +25,20 @@ mod sync {
2525
let TextAction { pos, ins, del } = action;
2626
if i % 2 == 0 {
2727
let mut txn = c1.txn().unwrap();
28-
t1.delete_with_txn(&mut txn, *pos, *del).unwrap();
29-
t1.insert_with_txn(&mut txn, *pos, ins).unwrap();
28+
t1.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode)
29+
.unwrap();
30+
t1.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode)
31+
.unwrap();
3032
txn.commit().unwrap();
3133

3234
let update = c1.export(ExportMode::updates(&c2.oplog_vv())).unwrap();
3335
c2.import(&update).unwrap();
3436
} else {
3537
let mut txn = c2.txn().unwrap();
36-
t2.delete_with_txn(&mut txn, *pos, *del).unwrap();
37-
t2.insert_with_txn(&mut txn, *pos, ins).unwrap();
38+
t2.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode)
39+
.unwrap();
40+
t2.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode)
41+
.unwrap();
3842
txn.commit().unwrap();
3943
let update = c2.export(ExportMode::updates(&c1.oplog_vv())).unwrap();
4044
c1.import(&update).unwrap();
@@ -127,7 +131,7 @@ mod run {
127131

128132
mod import {
129133
use criterion::Criterion;
130-
use loro_internal::{encoding::ExportMode, LoroDoc};
134+
use loro_internal::{cursor::PosType, encoding::ExportMode, LoroDoc};
131135

132136
#[allow(dead_code)]
133137
pub fn causal_iter(c: &mut Criterion) {
@@ -144,10 +148,10 @@ mod import {
144148
let text2 = c2.get_text("text");
145149
for _ in 0..500 {
146150
text1
147-
.insert_with_txn(&mut c1.txn().unwrap(), 0, "1")
151+
.insert_with_txn(&mut c1.txn().unwrap(), 0, "1", PosType::Unicode)
148152
.unwrap();
149153
text2
150-
.insert_with_txn(&mut c2.txn().unwrap(), 0, "2")
154+
.insert_with_txn(&mut c2.txn().unwrap(), 0, "2", PosType::Unicode)
151155
.unwrap();
152156
}
153157

‎crates/loro-internal/examples/automerge_x100.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use loro_internal::{loro::ExportMode, LoroDoc, VersionVector};
1+
use loro_internal::{cursor::PosType, loro::ExportMode, LoroDoc, VersionVector};
22
use std::hint::black_box;
33

44
fn main() {
@@ -15,8 +15,8 @@ fn main() {
1515
let mut v = VersionVector::new();
1616
for _ in 0..n {
1717
for TextAction { del, ins, pos } in actions.iter() {
18-
text.delete(*pos, *del).unwrap();
19-
text.insert(*pos, ins).unwrap();
18+
text.delete(*pos, *del, PosType::Unicode).unwrap();
19+
text.insert(*pos, ins, PosType::Unicode).unwrap();
2020
}
2121
loro.commit_then_renew();
2222
black_box(loro.export(ExportMode::updates(&v))).unwrap();

‎crates/loro-internal/examples/encoding.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use std::time::Instant;
22

33
use bench_utils::TextAction;
4-
use loro_internal::{loro::ExportMode, LoroDoc};
4+
use loro_internal::{cursor::PosType, loro::ExportMode, LoroDoc};
55

66
fn main() {
77
let actions = bench_utils::get_automerge_actions();
@@ -34,8 +34,10 @@ fn main() {
3434

3535
for TextAction { pos, ins, del } in actions.iter() {
3636
let mut txn = loro.txn().unwrap();
37-
text.delete_with_txn(&mut txn, *pos, *del).unwrap();
38-
text.insert_with_txn(&mut txn, *pos, ins).unwrap();
37+
text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode)
38+
.unwrap();
39+
text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode)
40+
.unwrap();
3941
txn.commit().unwrap();
4042
}
4143

‎crates/loro-internal/examples/encoding_parallel.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use bench_utils::{get_automerge_actions, TextAction};
2-
use loro_internal::{encoding::ExportMode, LoroDoc};
2+
use loro_internal::{cursor::PosType, encoding::ExportMode, LoroDoc};
33

44
// #[allow(dead_code)]
55
// fn parallel() {
@@ -69,8 +69,10 @@ fn main() {
6969
for TextAction { pos, ins, del } in actions.iter() {
7070
{
7171
let mut txn = loro.txn().unwrap();
72-
text.delete_with_txn(&mut txn, *pos, *del).unwrap();
73-
text.insert_with_txn(&mut txn, *pos, ins).unwrap();
72+
text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode)
73+
.unwrap();
74+
text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode)
75+
.unwrap();
7476
}
7577

7678
loro_b

‎crates/loro-internal/examples/encoding_refactored.rs‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use bench_utils::TextAction;
2-
use loro_internal::{loro::ExportMode, LoroDoc};
2+
use loro_internal::{cursor::PosType, loro::ExportMode, LoroDoc};
33
use std::hint::black_box;
44

55
fn main() {
@@ -17,8 +17,10 @@ fn log_size() {
1717
let mut txn = loro.txn().unwrap();
1818

1919
for TextAction { pos, ins, del } in actions.iter() {
20-
text.delete_with_txn(&mut txn, *pos, *del);
21-
text.insert_with_txn(&mut txn, *pos, ins);
20+
text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode)
21+
.unwrap();
22+
text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode)
23+
.unwrap();
2224
}
2325
txn.commit().unwrap();
2426
let snapshot = loro.export(ExportMode::Snapshot).unwrap();
@@ -46,8 +48,10 @@ fn log_size() {
4648

4749
for TextAction { pos, ins, del } in actions.iter() {
4850
let mut txn = loro.txn().unwrap();
49-
text.delete_with_txn(&mut txn, *pos, *del);
50-
text.insert_with_txn(&mut txn, *pos, ins);
51+
text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode)
52+
.unwrap();
53+
text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode)
54+
.unwrap();
5155
txn.commit().unwrap();
5256
}
5357
let snapshot = loro.export(ExportMode::Snapshot).unwrap();
@@ -71,8 +75,8 @@ fn bench_decode() {
7175

7276
for _ in 0..10 {
7377
for TextAction { pos, ins, del } in actions.iter() {
74-
text.delete(*pos, *del);
75-
text.insert(*pos, ins);
78+
text.delete(*pos, *del, PosType::Unicode).unwrap();
79+
text.insert(*pos, ins, PosType::Unicode).unwrap();
7680
}
7781
}
7882
let snapshot = loro.export(ExportMode::Snapshot).unwrap();
@@ -96,8 +100,10 @@ fn bench_decode_updates() {
96100
#[allow(warnings)]
97101
for TextAction { pos, ins, del } in actions.iter() {
98102
let mut txn = loro.txn().unwrap();
99-
text.delete_with_txn(&mut txn, *pos, *del);
100-
text.insert_with_txn(&mut txn, *pos, ins);
103+
text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode)
104+
.unwrap();
105+
text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode)
106+
.unwrap();
101107
txn.commit().unwrap();
102108
}
103109

‎crates/loro-internal/examples/event.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::sync::Arc;
33
use loro_internal::{
44
event::Diff,
55
handler::{Handler, ValueOrHandler},
6-
ListHandler, LoroDoc, MapHandler, TextHandler, ToJson, TreeHandler,
6+
cursor::PosType, ListHandler, LoroDoc, MapHandler, TextHandler, ToJson, TreeHandler,
77
};
88

99
fn main() {
@@ -29,7 +29,8 @@ fn main() {
2929
TextHandler::new_detached(),
3030
)
3131
.unwrap();
32-
text.insert(0, "created from event").unwrap();
32+
text.insert(0, "created from event", PosType::Unicode)
33+
.unwrap();
3334
}
3435
}
3536
ValueOrHandler::Value(value) => {

‎crates/loro-internal/examples/obs.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use bench_utils::TextAction;
22

3-
use loro_internal::LoroDoc;
3+
use loro_internal::{cursor::PosType, LoroDoc};
44

55
fn main() {
66
let actions = bench_utils::get_automerge_actions();
@@ -13,8 +13,10 @@ fn main() {
1313
// }));
1414
for TextAction { pos, ins, del } in actions.iter() {
1515
let mut txn = loro.txn().unwrap();
16-
text.delete_with_txn(&mut txn, *pos, *del).unwrap();
17-
text.insert_with_txn(&mut txn, *pos, ins).unwrap();
16+
text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode)
17+
.unwrap();
18+
text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode)
19+
.unwrap();
1820
}
1921

2022
text.diagnose();

‎crates/loro-internal/examples/pending.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use bench_utils::TextAction;
2-
use loro_internal::{encoding::ExportMode, LoroDoc, VersionVector};
2+
use loro_internal::{cursor::PosType, encoding::ExportMode, LoroDoc, VersionVector};
33

44
pub fn main() {
55
let loro = LoroDoc::default();
@@ -11,8 +11,10 @@ pub fn main() {
1111
for chunks in actions.chunks(action_length / 10) {
1212
for TextAction { pos, ins, del } in chunks {
1313
let mut txn = loro.txn().unwrap();
14-
text.delete_with_txn(&mut txn, *pos, *del).unwrap();
15-
text.insert_with_txn(&mut txn, *pos, ins).unwrap();
14+
text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode)
15+
.unwrap();
16+
text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode)
17+
.unwrap();
1618
let update = loro.export(ExportMode::updates(&latest_vv)).unwrap();
1719
updates.push(update);
1820
latest_vv = loro.oplog_vv();

‎crates/loro-internal/src/event.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -505,7 +505,7 @@ mod test {
505505
use itertools::Itertools;
506506
use loro_common::LoroValue;
507507

508-
use crate::{ApplyDiff, LoroDoc};
508+
use crate::{cursor::PosType, ApplyDiff, LoroDoc};
509509

510510
#[test]
511511
fn test_text_event() {
@@ -517,8 +517,10 @@ mod test {
517517
}));
518518
let mut txn = loro.txn().unwrap();
519519
let text = loro.get_text("id");
520-
text.insert_with_txn(&mut txn, 0, "hello").unwrap();
521-
text.insert_with_txn(&mut txn, 1, "223").unwrap();
520+
text.insert_with_txn(&mut txn, 0, "hello", PosType::Unicode)
521+
.unwrap();
522+
text.insert_with_txn(&mut txn, 1, "223", PosType::Unicode)
523+
.unwrap();
522524
txn.commit().unwrap();
523525
}
524526
}

0 commit comments

Comments
 (0)