feat: properly fold in UTF16 surrogates

Author: mzdun

CMakeLists.txt

@@ -92,6 +92,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 
 add_library(json STATIC
     src/json.cpp
+    src/string_reader.cpp
     include/json/json.hpp
     include/json/serdes.hpp)
 set_target_properties(json PROPERTIES

include/json/json.hpp

@@ -30,6 +30,23 @@ namespace json {
 
 	template <typename Value>
 	struct ordered_map {
+		ordered_map() = default;
+		ordered_map(ordered_map const&) = default;
+		ordered_map& operator=(ordered_map const&) = default;
+		ordered_map(ordered_map&&) = default;
+		ordered_map& operator=(ordered_map&&) = default;
+
+		using tree_map_t = std::map<string, Value>;
+		using tree_map_value_t = tree_map_t::value_type;
+
+		ordered_map(std::initializer_list<tree_map_value_t> tree_list)
+		    : tree_{tree_list} {
+			order_.reserve(tree_list.size());
+			for (auto const& [key, _] : tree_list) {
+				order_.push_back(key);
+			}
+		}
+
 		std::map<string, Value> const& items() const noexcept { return tree_; }
 		std::span<string const> keys() const noexcept { return order_; }
 
@@ -118,7 +135,7 @@ namespace json {
 		}
 
 	private:
-		std::map<string, Value> tree_{};
+		tree_map_t tree_{};
 		std::vector<string> order_{};
 	};
 

src/json.cpp

@@ -1,11 +1,13 @@
 // Copyright (c) 2021 midnightBITS
 // This code is licensed under MIT license (see LICENSE for details)
 
+#include <array>
 #include <cassert>
 #include <cmath>
 #include <iostream>
 #include <limits>
 #include <memory>
+#include <print>
 #include <stack>
 #include <json/json.hpp>
 
@@ -84,17 +86,18 @@ namespace json {
 		return from_json_impl(value, path);
 	}
 
+	node read_string(string_view::iterator&,
+	                 string_view::iterator const&,
+	                 read_mode mode);
+
 	namespace {
 		using uchar = unsigned char;
 		using iterator = string_view::iterator;
 
 		void skip_ws(iterator&, iterator const&, read_mode mode);
-		node read_string(iterator&, iterator const&, read_mode mode);
 		node read_number(iterator&, iterator const&, read_mode mode);
 		node read_keyword(iterator&, iterator const&, read_mode mode);
 
-		void encode(uint32_t ch, string& target);
-
 		void skip_ws(iterator& it, iterator const& end, read_mode mode) {
 			if (mode == read_mode::strict) {
 				while (it != end) {
@@ -438,154 +441,6 @@ namespace json {
 			return val;
 		}
 
-		unsigned hex_escape(iterator& it, iterator const& end) {
-			// sanity check for read_string
-			assert(*it == 'x');
-
-			++it;
-			auto const result = hex_digit(it, end);
-			if (result == INV_HEX) return INV_HEX_SQUARE;
-
-			auto const lower = hex_digit(it, end);
-			if (lower == INV_HEX) return INV_HEX_SQUARE;
-			return result * 16 + lower;  // NOLINT(readability-magic-numbers)
-		}
-
-		uint32_t unicode_escape(iterator& it, iterator const& end) {
-			static constexpr auto max = std::numeric_limits<uint32_t>::max();
-			// sanity check for read_string
-			assert(*it == 'u');
-
-			++it;
-			if (it == end) return max;
-			if (*it == '{') {
-				++it;
-				uint32_t val = 0;
-				while (it != end && *it != '}') {
-					auto const dig = hex_digit(it, end);
-					if (dig == INV_HEX) return max;
-					auto const overflow_guard = val;
-					val *= 16;  // NOLINT(readability-magic-numbers)
-					val += dig;
-					if (overflow_guard > val) return max;
-				}
-				if (it != end) ++it;
-				return val;
-			}
-
-			uint32_t val = 0;
-			for (int i = 0; i < 4; ++i) {
-				auto const dig = hex_digit(it, end);
-				if (dig == INV_HEX) return max;
-				val *= 16;  // NOLINT(readability-magic-numbers)
-				val += dig;
-			}
-			return val;
-		}
-
-		node read_string(iterator& it, iterator const& end, read_mode mode) {
-			// sanity check for value_reader::read and
-			// object_reader::read_object_key
-			assert(it != end && (*it == '"' || *it == '\''));
-
-			if (mode == read_mode::strict && *it == '\'') return {};
-
-			auto tmplt = *it;
-			++it;
-
-			string result{};
-			bool in_string = true;
-			bool in_escape = false;
-			while (it != end && in_string) {
-				if (in_escape) {
-					switch (*it) {
-						case '\r':
-							if (mode == read_mode::strict) return {};
-							++it;
-							if (it != end && *it == '\n') break;
-							--it;
-							break;
-						case '\n':
-							if (mode == read_mode::strict) return {};
-							++it;
-							if (it != end && *it == '\r') break;
-							--it;
-							break;
-						case 'b':
-							result.push_back('\b');
-							break;
-						case 'f':
-							result.push_back('\f');
-							break;
-						case 'n':
-							result.push_back('\n');
-							break;
-						case 'r':
-							result.push_back('\r');
-							break;
-						case 't':
-							result.push_back('\t');
-							break;
-						case 'v':
-							if (mode == read_mode::strict) return {};
-							result.push_back('\v');
-							break;
-						case 'x': {
-							if (mode == read_mode::strict) return {};
-							auto const val = hex_escape(it, end);
-							if (val > 255)  // NOLINT(readability-magic-numbers)
-								return {};
-							result.push_back(static_cast<string::value_type>(
-							    static_cast<uchar>(val)));
-							--it;
-							break;
-						}
-						case 'u': {
-							auto const val = unicode_escape(it, end);
-							if (val == std::numeric_limits<uint32_t>::max())
-								return {};
-							encode(val, result);
-							--it;
-							break;
-						}
-						case '"':
-						case '\\':
-						case '/':
-							result.push_back(*it);
-							break;
-						default:
-							if (mode == read_mode::strict) return {};
-							result.push_back(*it);
-							break;
-					}
-					++it;
-					in_escape = false;
-					continue;
-				}
-
-				if (*it == tmplt) {
-					++it;
-					in_string = false;
-					continue;
-				}
-
-				switch (*it) {
-					case '\\':
-						in_escape = true;
-						break;
-					default:
-						if (mode == read_mode::strict &&
-						    static_cast<unsigned char>(*it) < CHAR_SPACE)
-							return {};
-						result.push_back(*it);
-				}
-				++it;
-			}
-
-			if (mode == read_mode::strict && in_string) return {};
-			return node{std::move(result)};
-		}
-
 		node read_int(iterator& it,
 		              iterator const& end,
 		              unsigned base,
@@ -763,67 +618,6 @@ namespace json {
 			return {};
 		}
 
-		constexpr uchar firstByteMark[7] = {0x00, 0x00, 0xC0, 0xE0,
-		                                    0xF0, 0xF8, 0xFC};
-
-		enum : uint32_t {
-			UNI_SUR_HIGH_START = 0xD800,
-			UNI_SUR_HIGH_END = 0xDBFF,
-			UNI_SUR_LOW_START = 0xDC00,
-			UNI_SUR_LOW_END = 0xDFFF,
-			UNI_REPLACEMENT_CHAR = 0x0000FFFD,
-			UNI_MAX_BMP = 0x0000FFFF,
-			UNI_MAX_UTF16 = 0x0010FFFF,
-			UNI_MAX_LEGAL_UTF32 = 0x0010FFFF
-		};
-
-		constexpr uint32_t byteMask = 0xBF;
-		constexpr uint32_t byteMark = 0x80;
-
-		void encode(uint32_t ch, string& target) {
-			unsigned short bytesToWrite = 0;
-
-			/* Figure out how many bytes the result will require */
-			if (ch < 0x80u)  // NOLINT
-				bytesToWrite = 1;
-			else if (ch < 0x800u)  // NOLINT
-				bytesToWrite = 2;  // NOLINT
-			else if (ch >= UNI_SUR_HIGH_START &&
-			         ch <= UNI_SUR_LOW_END) {  // NOLINT
-				bytesToWrite = 3;              // NOLINT
-				ch = UNI_REPLACEMENT_CHAR;
-			} else if (ch < 0x10000u)  // NOLINT
-				bytesToWrite = 3;      // NOLINT
-			else if (ch <= UNI_MAX_LEGAL_UTF32)
-				bytesToWrite = 4;  // NOLINT
-			else {
-				bytesToWrite = 3;  // NOLINT
-				ch = UNI_REPLACEMENT_CHAR;
-			}
-
-			uchar mid[4];
-			uchar* midp = mid + sizeof(mid);
-			switch (bytesToWrite) { /* note: everything falls through. */
-				case 4:             // NOLINT
-					*--midp = static_cast<uchar>((ch | byteMark) & byteMask);
-					ch >>= 6;  // NOLINT
-					[[fallthrough]];
-				case 3:
-					*--midp = static_cast<uchar>((ch | byteMark) & byteMask);
-					ch >>= 6;  // NOLINT
-					[[fallthrough]];
-				case 2:
-					*--midp = static_cast<uchar>((ch | byteMark) & byteMask);
-					ch >>= 6;  // NOLINT
-					[[fallthrough]];
-				case 1:
-					*--midp =
-					    static_cast<uchar>(ch | firstByteMark[bytesToWrite]);
-			}
-			for (int i = 0; i < bytesToWrite; ++i)
-				target.push_back(static_cast<string::value_type>(*midp++));
-		}
-
 		struct size_judge {
 			size_t allowed_space;