Skip to content

Commit 078ac0d

Browse files
proostproost
committed
feat: introduce utf-8 validation options
1 parent 7f05c03 commit 078ac0d

3 files changed

Lines changed: 165 additions & 20 deletions

File tree

‎tuple/include/array_of_strings_sketch.hpp‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,25 +30,28 @@ namespace datasketches {
3030

3131
using array_of_strings = array<std::string>;
3232

33-
// default update policy for an array of strings
3433
class default_array_of_strings_update_policy {
3534
public:
36-
default_array_of_strings_update_policy() = default;
35+
explicit default_array_of_strings_update_policy(bool validate_utf8 = true);
3736

3837
array_of_strings create() const;
3938

4039
void update(array_of_strings& array, const array_of_strings& input) const;
4140

4241
void update(array_of_strings& array, const array_of_strings* input) const;
42+
43+
private:
44+
bool validate_utf8_;
45+
static void check_utf8(const std::string& value);
4346
};
4447

45-
// serializer/deserializer for an array of strings
46-
// Requirements: all strings must be valid UTF-8 and array size must be <= 127.
48+
// Requirements: array size must be <= 127.
4749
template<typename Allocator = std::allocator<array_of_strings>>
48-
struct default_array_of_strings_serde {
50+
class default_array_of_strings_serde {
51+
public:
4952
using summary_allocator = typename std::allocator_traits<Allocator>::template rebind_alloc<array_of_strings>;
5053

51-
explicit default_array_of_strings_serde(const Allocator& allocator = Allocator());
54+
explicit default_array_of_strings_serde(const Allocator& allocator = Allocator(), bool validate_utf8 = false);
5255

5356
void serialize(std::ostream& os, const array_of_strings* items, unsigned num) const;
5457
void deserialize(std::istream& is, array_of_strings* items, unsigned num) const;
@@ -58,6 +61,7 @@ struct default_array_of_strings_serde {
5861

5962
private:
6063
summary_allocator summary_allocator_;
64+
bool validate_utf8_;
6165
static void check_num_nodes(uint8_t num_nodes);
6266
static uint32_t compute_total_bytes(const array_of_strings& item);
6367
static void check_utf8(const std::string& value);

‎tuple/include/array_of_strings_sketch_impl.hpp‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828

2929
namespace datasketches {
3030

31+
inline default_array_of_strings_update_policy::default_array_of_strings_update_policy(bool validate_utf8):
32+
validate_utf8_(validate_utf8) {}
33+
3134
inline array_of_strings default_array_of_strings_update_policy::create() const {
3235
return array_of_strings(0, "");
3336
}
@@ -36,6 +39,9 @@ inline void default_array_of_strings_update_policy::update(
3639
array_of_strings& array, const array_of_strings& input
3740
) const {
3841
const auto length = static_cast<size_t>(input.size());
42+
if (validate_utf8_) {
43+
for (size_t i = 0; i < length; ++i) check_utf8(input[i]);
44+
}
3945
array = array_of_strings(static_cast<uint8_t>(length), "");
4046
for (size_t i = 0; i < length; ++i) array[i] = input[i];
4147
}
@@ -48,10 +54,19 @@ inline void default_array_of_strings_update_policy::update(
4854
return;
4955
}
5056
const auto length = static_cast<size_t>(input->size());
57+
if (validate_utf8_) {
58+
for (size_t i = 0; i < length; ++i) check_utf8((*input)[i]);
59+
}
5160
array = array_of_strings(static_cast<uint8_t>(length), "");
5261
for (size_t i = 0; i < length; ++i) array[i] = (*input)[i];
5362
}
5463

64+
inline void default_array_of_strings_update_policy::check_utf8(const std::string& value) {
65+
if (!utf8::is_valid(value.begin(), value.end())) {
66+
throw std::runtime_error("array_of_strings contains invalid UTF-8 string");
67+
}
68+
}
69+
5570
inline uint64_t hash_array_of_strings_key(const array_of_strings& key) {
5671
// Matches Java Util.PRIME for ArrayOfStrings key hashing.
5772
static constexpr uint64_t STRING_ARR_HASH_SEED = 0x7A3CCA71ULL;
@@ -102,8 +117,8 @@ auto compact_array_of_strings_tuple_sketch<Allocator>::deserialize(
102117
}
103118

104119
template<typename Allocator>
105-
default_array_of_strings_serde<Allocator>::default_array_of_strings_serde(const Allocator& allocator):
106-
summary_allocator_(allocator) {}
120+
default_array_of_strings_serde<Allocator>::default_array_of_strings_serde(const Allocator& allocator, bool validate_utf8):
121+
summary_allocator_(allocator), validate_utf8_(validate_utf8) {}
107122

108123
template<typename Allocator>
109124
void default_array_of_strings_serde<Allocator>::serialize(
@@ -116,7 +131,7 @@ void default_array_of_strings_serde<Allocator>::serialize(
116131
write(os, num_nodes);
117132
const std::string* data = items[i].data();
118133
for (uint8_t j = 0; j < num_nodes; ++j) {
119-
check_utf8(data[j]);
134+
if (validate_utf8_) check_utf8(data[j]);
120135
const uint32_t length = static_cast<uint32_t>(data[j].size());
121136
write(os, length);
122137
os.write(data[j].data(), length);
@@ -143,7 +158,7 @@ void default_array_of_strings_serde<Allocator>::deserialize(
143158
is.read(&value[0], length);
144159
if (!is) throw std::runtime_error("array_of_strings stream read failed");
145160
}
146-
check_utf8(value);
161+
if (validate_utf8_) check_utf8(value);
147162
array[j] = std::move(value);
148163
}
149164
summary_allocator alloc(summary_allocator_);
@@ -166,7 +181,7 @@ size_t default_array_of_strings_serde<Allocator>::serialize(
166181
bytes_written += copy_to_mem(num_nodes, ptr8 + bytes_written);
167182
const std::string* data = items[i].data();
168183
for (uint8_t j = 0; j < num_nodes; ++j) {
169-
check_utf8(data[j]);
184+
if (validate_utf8_) check_utf8(data[j]);
170185
const uint32_t length = static_cast<uint32_t>(data[j].size());
171186

172187
bytes_written += copy_to_mem(length, ptr8 + bytes_written);
@@ -200,7 +215,7 @@ size_t default_array_of_strings_serde<Allocator>::deserialize(
200215
if (length != 0) {
201216
bytes_read += copy_from_mem(ptr8 + bytes_read, &value[0], length);
202217
}
203-
check_utf8(value);
218+
if (validate_utf8_) check_utf8(value);
204219
array[j] = std::move(value);
205220
}
206221
summary_allocator alloc(summary_allocator_);

‎tuple/test/array_of_strings_sketch_test.cpp‎

Lines changed: 134 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,50 @@ TEST_CASE("aos update policy", "[tuple_sketch]") {
7878
input[1] = "changed";
7979
REQUIRE(values[1] == "second");
8080
}
81+
82+
SECTION("invalid utf8 rejected by default") {
83+
auto values = policy.create();
84+
85+
array_of_strings input(1, "", std::allocator<std::string>());
86+
input[0] = std::string("\xC3\x28", 2);
87+
REQUIRE_THROWS_WITH(
88+
policy.update(values, input),
89+
Catch::Matchers::Contains("invalid UTF-8 string")
90+
);
91+
}
92+
93+
SECTION("invalid utf8 rejected via pointer by default") {
94+
auto values = policy.create();
95+
96+
array_of_strings input(1, "", std::allocator<std::string>());
97+
input[0] = std::string("\xC3\x28", 2);
98+
REQUIRE_THROWS_WITH(
99+
policy.update(values, &input),
100+
Catch::Matchers::Contains("invalid UTF-8 string")
101+
);
102+
}
103+
104+
SECTION("invalid utf8 allowed when validation disabled") {
105+
default_array_of_strings_update_policy no_validate_policy(false);
106+
auto values = no_validate_policy.create();
107+
108+
array_of_strings input(1, "", std::allocator<std::string>());
109+
input[0] = std::string("\xC3\x28", 2);
110+
REQUIRE_NOTHROW(no_validate_policy.update(values, input));
111+
REQUIRE(values.size() == 1);
112+
REQUIRE(values[0] == std::string("\xC3\x28", 2));
113+
}
114+
115+
SECTION("invalid utf8 allowed via pointer when validation disabled") {
116+
default_array_of_strings_update_policy no_validate_policy(false);
117+
auto values = no_validate_policy.create();
118+
119+
array_of_strings input(1, "", std::allocator<std::string>());
120+
input[0] = std::string("\xC3\x28", 2);
121+
REQUIRE_NOTHROW(no_validate_policy.update(values, &input));
122+
REQUIRE(values.size() == 1);
123+
REQUIRE(values[0] == std::string("\xC3\x28", 2));
124+
}
81125
}
82126

83127
TEST_CASE("aos sketch update", "[tuple_sketch]") {
@@ -254,26 +298,108 @@ TEST_CASE("aos sketch: serialize deserialize", "[tuple_sketch]") {
254298
}
255299

256300
TEST_CASE("aos serde validation", "[tuple_sketch]") {
257-
default_array_of_strings_serde<> serde;
301+
default_array_of_strings_serde<> serde_no_validate;
302+
default_array_of_strings_serde<> serde_validate(std::allocator<array_of_strings>(), true);
258303

259-
SECTION("invalid utf8 rejected") {
260-
array_of_strings array(1, "", std::allocator<std::string>());
261-
const std::string invalid_utf8("\xC3\x28", 2);
262-
array[0] = invalid_utf8;
304+
auto make_invalid_array = []() {
305+
array_of_strings arr(1, "", std::allocator<std::string>());
306+
arr[0] = std::string("\xC3\x28", 2);
307+
return arr;
308+
};
309+
310+
auto serialize_to_stream = [&](const array_of_strings& arr) {
263311
std::stringstream ss;
264312
ss.exceptions(std::ios::failbit | std::ios::badbit);
313+
serde_no_validate.serialize(ss, &arr, 1);
314+
return ss;
315+
};
316+
317+
auto serialize_to_bytes = [&](const array_of_strings& arr) {
318+
const size_t capacity = serde_no_validate.size_of_item(arr) + 64;
319+
std::vector<uint8_t> buf(capacity);
320+
const size_t written = serde_no_validate.serialize(buf.data(), capacity, &arr, 1);
321+
buf.resize(written);
322+
return buf;
323+
};
324+
325+
SECTION("stream serialize: invalid utf8 allowed by default") {
326+
auto arr = make_invalid_array();
327+
std::stringstream ss;
328+
ss.exceptions(std::ios::failbit | std::ios::badbit);
329+
REQUIRE_NOTHROW(serde_no_validate.serialize(ss, &arr, 1));
330+
}
331+
332+
SECTION("stream serialize: invalid utf8 rejected when validation enabled") {
333+
auto arr = make_invalid_array();
334+
std::stringstream ss;
335+
ss.exceptions(std::ios::failbit | std::ios::badbit);
336+
REQUIRE_THROWS_WITH(
337+
serde_validate.serialize(ss, &arr, 1),
338+
Catch::Matchers::Contains("invalid UTF-8")
339+
);
340+
}
341+
342+
SECTION("bytes serialize: invalid utf8 allowed by default") {
343+
auto arr = make_invalid_array();
344+
const size_t capacity = serde_no_validate.size_of_item(arr) + 64;
345+
std::vector<uint8_t> buf(capacity);
346+
REQUIRE_NOTHROW(serde_no_validate.serialize(buf.data(), capacity, &arr, 1));
347+
}
348+
349+
SECTION("bytes serialize: invalid utf8 rejected when validation enabled") {
350+
auto arr = make_invalid_array();
351+
const size_t capacity = serde_validate.size_of_item(arr) + 64;
352+
std::vector<uint8_t> buf(capacity);
353+
REQUIRE_THROWS_WITH(
354+
serde_validate.serialize(buf.data(), capacity, &arr, 1),
355+
Catch::Matchers::Contains("invalid UTF-8")
356+
);
357+
}
358+
359+
SECTION("stream deserialize: invalid utf8 allowed by default") {
360+
auto arr = make_invalid_array();
361+
auto ss = serialize_to_stream(arr);
362+
array_of_strings result(0, "");
363+
REQUIRE_NOTHROW(serde_no_validate.deserialize(ss, &result, 1));
364+
REQUIRE(result.size() == 1);
365+
REQUIRE(result[0] == std::string("\xC3\x28", 2));
366+
}
367+
368+
SECTION("stream deserialize: invalid utf8 rejected when validation enabled") {
369+
auto arr = make_invalid_array();
370+
auto ss = serialize_to_stream(arr);
371+
array_of_strings result(0, "");
372+
REQUIRE_THROWS_WITH(
373+
serde_validate.deserialize(ss, &result, 1),
374+
Catch::Matchers::Contains("invalid UTF-8")
375+
);
376+
}
377+
378+
SECTION("bytes deserialize: invalid utf8 allowed by default") {
379+
auto arr = make_invalid_array();
380+
auto buf = serialize_to_bytes(arr);
381+
array_of_strings result(0, "");
382+
REQUIRE_NOTHROW(serde_no_validate.deserialize(buf.data(), buf.size(), &result, 1));
383+
REQUIRE(result.size() == 1);
384+
REQUIRE(result[0] == std::string("\xC3\x28", 2));
385+
}
386+
387+
SECTION("bytes deserialize: invalid utf8 rejected when validation enabled") {
388+
auto arr = make_invalid_array();
389+
auto buf = serialize_to_bytes(arr);
390+
array_of_strings result(0, "");
265391
REQUIRE_THROWS_WITH(
266-
serde.serialize(ss, &array, 1),
392+
serde_validate.deserialize(buf.data(), buf.size(), &result, 1),
267393
Catch::Matchers::Contains("invalid UTF-8")
268394
);
269395
}
270396

271397
SECTION("too many nodes rejected") {
272-
array_of_strings array(128, "", std::allocator<std::string>());
398+
array_of_strings arr(128, "", std::allocator<std::string>());
273399
std::stringstream ss;
274400
ss.exceptions(std::ios::failbit | std::ios::badbit);
275401
REQUIRE_THROWS_WITH(
276-
serde.serialize(ss, &array, 1),
402+
serde_no_validate.serialize(ss, &arr, 1),
277403
Catch::Matchers::Contains("size exceeds 127")
278404
);
279405
}

0 commit comments

Comments
 (0)