-
Notifications
You must be signed in to change notification settings - Fork 11
Expand file tree
/
Copy pathfix_data.rs
More file actions
203 lines (175 loc) · 6.79 KB
/
fix_data.rs
File metadata and controls
203 lines (175 loc) · 6.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
use core::ffi::c_int;
use alloc::format;
use alloc::string::String;
use crate::create_sqlite_optional_text_fn;
use crate::error::{PSResult, PowerSyncError};
use crate::schema::inspection::ExistingTable;
use crate::utils::SqlBuffer;
use powersync_sqlite_nostd::{self as sqlite, ColumnType, Value};
use powersync_sqlite_nostd::{Connection, Context, ResultCode};
use crate::ext::SafeManagedStmt;
// Apply a data migration to fix any existing data affected by the issue
// fixed in v0.3.5.
//
// The issue was that the `ps_updated_rows` table was not being populated
// with remove operations in some cases. This causes the rows to be removed
// from ps_oplog, but not from the ps_data__tables, resulting in dangling rows.
//
// The fix here is to find these dangling rows, and add them to ps_updated_rows.
// The next time the sync_local operation is run, these rows will be removed.
pub fn apply_v035_fix(db: *mut sqlite::sqlite3) -> Result<i64, PowerSyncError> {
// language=SQLite
let statement = db
.prepare_v2("SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data__*'")
.into_db_result(db)?;
while statement.step()? == ResultCode::ROW {
let full_name = statement.column_text(0)?;
let Some((short_name, _)) = ExistingTable::external_name(full_name) else {
continue;
};
let quoted = SqlBuffer::quote_identifier(full_name);
// language=SQLite
let statement = db.prepare_v2(&format!(
"
INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id)
SELECT ?1, id FROM {}
WHERE NOT EXISTS (
SELECT 1 FROM ps_oplog
WHERE row_type = ?1 AND row_id = {}.id
);",
quoted, quoted
))?;
statement.bind_text(1, short_name, sqlite::Destructor::STATIC)?;
statement.exec()?;
}
Ok(1)
}
/// Older versions of the JavaScript SDK for PowerSync used to encode the subkey in oplog data
/// entries as JSON.
///
/// It wasn't supposed to do that, since the keys are regular strings already. To make databases
/// created with those SDKs compatible with other SDKs or the sync client implemented in the core
/// extensions, a migration is necessary. Since this migration is only relevant for the JS SDK, it
/// is mostly implemented there. However, the helper function to remove the key encoding is
/// implemented here because user-defined functions are expensive on JavaScript.
fn remove_duplicate_key_encoding(key: &str) -> Option<String> {
// Acceptable format: <type>/<id>/<subkey>
// Inacceptable format: <type>/<id>/"<subkey>"
// This is a bit of a tricky conversion because both type and id can contain slashes and quotes.
// However, the subkey is either a UUID value or a `<table>/UUID` value - so we know it can't
// end in a quote unless the improper encoding was used.
if !key.ends_with('"') {
return None;
}
// Since the subkey is JSON-encoded, find the start quote by going backwards.
let mut chars = key.char_indices();
chars.next_back()?; // Skip the quote ending the string
enum FoundStartingQuote {
HasQuote { index: usize },
HasBackslachThenQuote { quote_index: usize },
}
let mut state: Option<FoundStartingQuote> = None;
let found_starting_quote = loop {
if let Some((i, char)) = chars.next_back() {
state = match state {
Some(FoundStartingQuote::HasQuote { index }) => {
if char == '\\' {
// We've seen a \" pattern, not the start of the string
Some(FoundStartingQuote::HasBackslachThenQuote { quote_index: index })
} else {
break Some(index);
}
}
Some(FoundStartingQuote::HasBackslachThenQuote { quote_index }) => {
if char == '\\' {
// \\" pattern, the quote is unescaped
break Some(quote_index);
} else {
None
}
}
None => {
if char == '"' {
Some(FoundStartingQuote::HasQuote { index: i })
} else {
None
}
}
}
} else {
break None;
}
}?;
let before_json = &key[..found_starting_quote];
let mut result: String = serde_json::from_str(&key[found_starting_quote..]).ok()?;
result.insert_str(0, before_json);
Some(result)
}
fn powersync_remove_duplicate_key_encoding_impl(
_ctx: *mut sqlite::context,
args: &[*mut sqlite::value],
) -> Result<Option<String>, PowerSyncError> {
let arg = args.get(0).ok_or(ResultCode::MISUSE)?;
if arg.value_type() != ColumnType::Text {
return Err(ResultCode::MISMATCH.into());
}
return Ok(remove_duplicate_key_encoding(arg.text()));
}
create_sqlite_optional_text_fn!(
powersync_remove_duplicate_key_encoding,
powersync_remove_duplicate_key_encoding_impl,
"powersync_remove_duplicate_key_encoding"
);
pub fn register(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> {
db.create_function_v2(
"powersync_remove_duplicate_key_encoding",
1,
sqlite::UTF8 | sqlite::DETERMINISTIC,
None,
Some(powersync_remove_duplicate_key_encoding),
None,
None,
None,
)?;
Ok(())
}
#[cfg(test)]
mod test {
use super::remove_duplicate_key_encoding;
fn assert_unaffected(source: &str) {
assert!(matches!(remove_duplicate_key_encoding(source), None));
}
#[test]
fn does_not_change_unaffected_keys() {
assert_unaffected("object_type/object_id/subkey");
assert_unaffected("object_type/object_id/null");
// Object type and ID could technically contain quotes and forward slashes
assert_unaffected(r#""object"/"type"/subkey"#);
assert_unaffected("object\"/type/object\"/id/subkey");
// Invalid key, but we shouldn't crash
assert_unaffected("\"key\"");
}
#[test]
fn removes_quotes() {
assert_eq!(
remove_duplicate_key_encoding("foo/bar/\"baz\"").unwrap(),
"foo/bar/baz",
);
assert_eq!(
remove_duplicate_key_encoding(r#"foo/bar/"nested/subkey""#).unwrap(),
"foo/bar/nested/subkey"
);
assert_eq!(
remove_duplicate_key_encoding(r#"foo/bar/"escaped\"key""#).unwrap(),
"foo/bar/escaped\"key"
);
assert_eq!(
remove_duplicate_key_encoding(r#"foo/bar/"escaped\\key""#).unwrap(),
"foo/bar/escaped\\key"
);
assert_eq!(
remove_duplicate_key_encoding(r#"foo/bar/"/\\"subkey""#).unwrap(),
"foo/bar/\"/\\\\subkey"
);
}
}