Skip to content

Commit 509f437

Browse files
adriwebadriweb
committed
Evo: handle even more UCS-2 ranges and direct token mappings.
Credits to Zeroko mostly for painfully testing ranges.
1 parent fb1a740 commit 509f437

2 files changed

Lines changed: 203 additions & 19 deletions

File tree

‎src/EvoFormat.cpp‎

Lines changed: 87 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include <sstream>
1616
#include <stdexcept>
1717
#include <unordered_map>
18+
#include <utility>
1819

1920
#include "TIVarTypes.h"
2021
#include "TypeHandlers/TypeHandlers.h"
@@ -58,9 +59,24 @@ namespace tivars::EvoFormat
5859

5960
bool is_displayable_ucs2_scalar(uint16_t codepoint)
6061
{
61-
return (codepoint >= 0x0020 && codepoint <= 0x007E)
62-
|| (codepoint >= 0x00A0 && codepoint <= 0x00FF)
63-
|| codepoint == 0x0177;
62+
static constexpr std::pair<uint16_t, uint16_t> acceptedRanges[] = {
63+
{0x0020, 0x007E}, {0x00A0, 0x00FF}, {0x0177, 0x0177}, {0x0394, 0x0394},
64+
{0x03A3, 0x03A3}, {0x03A9, 0x03A9}, {0x03B1, 0x03B5}, {0x03B8, 0x03B8},
65+
{0x03BB, 0x03BC}, {0x03C0, 0x03C1}, {0x03C3, 0x03C4}, {0x03C6, 0x03C7},
66+
{0x2010, 0x2010}, {0x2026, 0x2026}, {0x2070, 0x2070}, {0x2074, 0x2079},
67+
{0x2080, 0x2089}, {0x2122, 0x2122}, {0x2190, 0x2193}, {0x221A, 0x221A},
68+
{0x2220, 0x2220}, {0x222B, 0x222B}, {0x2260, 0x2260}, {0x2264, 0x2265},
69+
{0x238C, 0x238C}, {0x25A0, 0x25A0}, {0x25AB, 0x25AB}, {0x25B2, 0x25B2},
70+
{0x25B6, 0x25B6}, {0x25B8, 0x25B8}, {0x25BC, 0x25BC}, {0x25C0, 0x25C0},
71+
{0x25C2, 0x25C2}, {0xF000, 0xF032}, {0xF038, 0xF03A}, {0xF041, 0xF04D},
72+
{0xF04F, 0xF058}, {0xF05B, 0xF061},
73+
};
74+
75+
return std::ranges::any_of(acceptedRanges, [codepoint](const auto& range)
76+
{
77+
const auto& [first, last] = range;
78+
return codepoint >= first && codepoint <= last;
79+
});
6480
}
6581

6682
bool utf8_to_single_codepoint(const std::string& text, uint16_t& codepoint)
@@ -752,13 +768,45 @@ static const char* evo_token_name(uint16_t token)
752768
}
753769

754770
static bool direct_legacy_token_for_evo(uint16_t evoToken, uint16_t& legacyToken);
771+
static bool direct_legacy_payload_for_evo(uint16_t evoToken, data_t& payload);
755772
static bool direct_evo_token_for_legacy(uint16_t legacyToken, uint16_t& evoToken);
756773
static void append_evo_token(data_t& out, uint16_t evoToken);
757774
static bool legacy_token_to_evo_ucs2(uint16_t legacyToken, uint16_t& evoToken);
758775

776+
static const std::unordered_map<uint16_t, std::string>& evo_private_display_aliases()
777+
{
778+
static const std::unordered_map<uint16_t, std::string> aliases = {
779+
{0xF000, "ᴇ"}, {0xF001, "E"}, {0xF002, "e"}, {0xF003, "𝙵"},
780+
{0xF004, "𝑖"}, {0xF005, "ʟ"}, {0xF006, "𝗡"}, {0xF007, "𝑛"},
781+
{0xF008, "p̂"}, {0xF009, "ʳ"}, {0xF00A, "ᵀ"}, {0xF00B, "ᴛ"},
782+
{0xF00C, "ˣ"}, {0xF00D, "x̄"}, {0xF00E, "ȳ"}, {0xF00F, "⁺"},
783+
{0xF010, "⁻"}, {0xF011, "⁻¹"}, {0xF012, "₁₀"},{0xF013, "²"},
784+
{0xF014, "³"}, {0xF015, "⧸"}, {0xF016, "⟋"}, {0xF017, "␣"},
785+
{0xF018, "⁄"}, {0xF019, "`"},
786+
{0xF01A, "⸩"}, {0xF01B, "🡅"}, {0xF01C, "🡇"}, {0xF01D, "🠺"},
787+
{0xF01E, "↑"}, {0xF01F, "↓"},
788+
{0xF020, "⌸"}, {0xF021, "▮"}, {0xF022, "↑"},
789+
{0xF023, "A"}, {0xF024, "a"}, {0xF025, "_"}, {0xF026, "↑͟"},
790+
{0xF027, "A͟"}, {0xF028, "a͟"}, {0xF029, "░"}, {0xF02A, "⬚"},
791+
{0xF02B, "╲"}, {0xF02C, "＼"}, {0xF02D, "◥"}, {0xF02E, "◣"},
792+
{0xF02F, "⌕"}, {0xF030, "⁰"}, {0xF031, "⧵"}, {0xF032, "⧹"},
793+
{0xF038, "⬩"}, {0xF039, "⎵"}, {0xF03A, "🔒"}, {0xF041, "⋅"},
794+
{0xF042, "₀"}, {0xF043, "₁"}, {0xF044, "₂"}, {0xF045, "₃"},
795+
{0xF046, "₄"}, {0xF047, "₅"}, {0xF048, "₆"}, {0xF049, "₇"},
796+
{0xF04A, "₈"}, {0xF04B, "₉"}, {0xF04C, "□"},
797+
{0xF04F, "⬌"}, {0xF050, "▯"}, {0xF051, "⸋"}, {0xF052, "𝅆"},
798+
{0xF053, "ᵍ"}, {0xF054, "▫"}, {0xF055, "ᵃ"}, {0xF056, "🔒"},
799+
{0xF057, "◣̏"}, {0xF058, "◥̤"},
800+
{0xF05B, "Β"}, {0xF05C, "Ε"}, {0xF05D, "Ｆ"},
801+
{0xF05E, "‹"}, {0xF05F, "›"}, {0xF060, "≤"}, {0xF061, "≥"}
802+
};
803+
return aliases;
804+
}
805+
759806
static std::string evo_token_to_string(uint16_t token)
760807
{
761808
if (token == 0x0000) return "";
809+
if (const auto it = evo_private_display_aliases().find(token); it != evo_private_display_aliases().end()) return it->second;
762810
if (is_displayable_ucs2_scalar(token)) return utf8_from_codepoint(token);
763811
if (token >= 0xE800 && token <= 0xE819) return std::string(1, static_cast<char>('A' + (token - 0xE800)));
764812
if (token == 0xE81A) return "θ";
@@ -770,6 +818,9 @@ static std::string evo_token_to_string(uint16_t token)
770818
if (token == 0xE41A) return "'";
771819
if (token == 0xE424) return "ᵍ";
772820
if (token == 0xE589) return "Grad";
821+
if (token == 0xE9D6) return "►ʳ";
822+
if (token == 0xE9D7) return "►ᵍ";
823+
if (token == 0xE9D8) return "►º";
773824
if (token >= 0xE850 && token <= 0xE85B)
774825
{
775826
const uint16_t idx = static_cast<uint16_t>((token - 0xE850) / 2 + 1);
@@ -946,6 +997,28 @@ static bool legacy_payload_for_evo_ucs2(uint16_t evoToken, data_t& payload)
946997
}
947998
}
948999

1000+
static bool direct_legacy_payload_for_evo(uint16_t evoToken, data_t& payload)
1001+
{
1002+
payload.clear();
1003+
1004+
// ►{angle} conv token
1005+
if (evoToken == 0xE9D6 || evoToken == 0xE9D7 || evoToken == 0xE9D8)
1006+
{
1007+
append_legacy_token(payload, 0xBBEC);
1008+
append_legacy_token(payload, evoToken == 0xE9D6 ? 0x0A : evoToken == 0xE9D7 ? 0xAF : 0x0B);
1009+
return true;
1010+
}
1011+
1012+
uint16_t legacyToken = 0;
1013+
if (!direct_legacy_token_for_evo(evoToken, legacyToken))
1014+
{
1015+
return false;
1016+
}
1017+
1018+
append_legacy_token(payload, legacyToken);
1019+
return true;
1020+
}
1021+
9491022
static bool legacy_token_to_evo_ucs2(uint16_t legacyToken, uint16_t& evoToken)
9501023
{
9511024
const std::string text = TypeHandlers::TH_Tokenized::oneTokenBytesToString(legacyToken);
@@ -1092,7 +1165,7 @@ static bool direct_legacy_token_for_evo(uint16_t evoToken, uint16_t& legacyToken
10921165
{0x03B4, 0xBBA3}, {0x03B5, 0xBBA4}, {0x03BB, 0xBBA5}, {0x03BC, 0xBBA6},
10931166
{0x03C0, 0xBBA7}, {0x03C1, 0xBBA8}, {0x03A3, 0xBBA9}, {0x03A6, 0xBBAB},
10941167
{0x03A9, 0xBBAC}, {0x03C7, 0xBBAE}, {0x007C, 0xBBD8}, {0x2026, 0xBBDB},
1095-
{0x00D7, 0xBBF0}, {0x222B, 0xBBF1}, {0x2338, 0xBBF5},
1168+
{0x00D7, 0xBBF0}, {0x222B, 0xBBF1},
10961169
{0x007E, 0xBBCF}, {0x03C3, 0xBBCB}, {0x03C4, 0xBBCC}, {0x00CD, 0xBBCD},
10971170
{0x0040, 0xBBD1}, {0x0023, 0xBBD2}, {0x0024, 0xBBD3}, {0x0026, 0xBBD4},
10981171
{0x003B, 0xBBD6}, {0x005C, 0xBBD7}, {0x0025, 0xBBDA}, {0x2220, 0xBBDC},
@@ -1105,6 +1178,7 @@ static bool direct_legacy_token_for_evo(uint16_t evoToken, uint16_t& legacyToken
11051178
{0xE5BD, 0x7F}, {0xE5BE, 0x80}, {0xE5BF, 0x81}, {0xE5C0, 0xEF73},
11061179
{0xE5C1, 0xEF74}, {0xE5C2, 0xEF75},
11071180
{0xE4F9, 0xBB57}, {0xE593, 0xBB64}, {0xE6C6, 0xE8}, {0xE6C7, 0xE7},
1181+
{0xE6AE, 0xEF79},
11081182
{0xE900, 0x6201}, {0xE901, 0x6202}, {0xE902, 0x6203}, {0xE903, 0x6204},
11091183
{0xE904, 0x6205}, {0xE905, 0x6206}, {0xE906, 0x6207}, {0xE907, 0x6208},
11101184
{0xE908, 0x6209}, {0xE909, 0x620A}, {0xE90A, 0x620B}, {0xE90B, 0x620C},
@@ -1121,6 +1195,9 @@ static bool direct_legacy_token_for_evo(uint16_t evoToken, uint16_t& legacyToken
11211195
{0xE932, 0x6234}, {0xE933, 0x6235}, {0xE934, 0x6236}, {0xE935, 0x6237},
11221196
{0xE936, 0x6238}, {0xE937, 0x6239}, {0xE938, 0x623A}, {0xE939, 0x623B},
11231197
{0xE93A, 0x623C}, {0xE980, 0x6304}, {0xE981, 0x6305}, {0xE982, 0x6332},
1198+
{0xE93B, 0x6203}, {0xE93C, 0x622B}, {0xE93D, 0x622E}, {0xE93E, 0xBBA6},
1199+
{0xE941, 0x622D}, {0xE942, 0x6230}, {0xE943, 0x6206}, {0xE944, 0x622C},
1200+
{0xE945, 0x622F}, {0xE946, 0xBBCB}, {0xE95C, 0x6227},
11241201
{0xE983, 0x6306}, {0xE984, 0x6307}, {0xE985, 0x6308}, {0xE986, 0x6309},
11251202
{0xE987, 0x6333}, {0xE98F, 0x630A}, {0xE990, 0x630B}, {0xE991, 0x6302},
11261203
{0xE992, 0x6336}, {0xE993, 0x630C}, {0xE994, 0x630D}, {0xE995, 0x6303},
@@ -1444,6 +1521,7 @@ static bool direct_legacy_token_for_evo(uint16_t evoToken, uint16_t& legacyToken
14441521
{0xE6C5, 0xEF31},
14451522
{0xE81B, 0x0072},
14461523
{0xE81C, 0x005F},
1524+
{0xF012, 0xBBEA},
14471525
};
14481526

14491527
const auto it = direct.find(evoToken);
@@ -1588,7 +1666,7 @@ static bool direct_evo_token_for_legacy(uint16_t legacyToken, uint16_t& evoToken
15881666
{0xBBA3, 0x03B4}, {0xBBA4, 0x03B5}, {0xBBA5, 0x03BB}, {0xBBA6, 0x03BC},
15891667
{0xBBA7, 0x03C0}, {0xBBA8, 0x03C1}, {0xBBA9, 0x03A3}, {0xBBAB, 0x03A6},
15901668
{0xBBAC, 0x03A9}, {0xBBAE, 0x03C7}, {0xBBD8, 0x007C}, {0xBBDB, 0x2026},
1591-
{0xBBF0, 0x00D7}, {0xBBF1, 0x222B}, {0xBBF5, 0x2338},
1669+
{0xBBF0, 0x00D7}, {0xBBF1, 0x222B},
15921670
{0xBBCF, 0x007E}, {0xBBCB, 0x03C3}, {0xBBCC, 0x03C4}, {0xBBCD, 0x00CD},
15931671
{0xBBD1, 0x0040}, {0xBBD2, 0x0023}, {0xBBD3, 0x0024}, {0xBBD4, 0x0026},
15941672
{0xBBD5, 0x0060}, {0xBBD6, 0x003B}, {0xBBD7, 0x005C}, {0xBBDA, 0x0025}, {0xBBDC, 0x2220},
@@ -1600,6 +1678,7 @@ static bool direct_evo_token_for_legacy(uint16_t legacyToken, uint16_t& evoToken
16001678
{0x7F, 0xE5BD}, {0x80, 0xE5BE}, {0x81, 0xE5BF}, {0xEF73, 0xE5C0},
16011679
{0xEF74, 0xE5C1}, {0xEF75, 0xE5C2},
16021680
{0xBB57, 0xE4F9}, {0xBB64, 0xE593}, {0xE8, 0xE6C6}, {0xE7, 0xE6C7},
1681+
{0xEF79, 0xE6AE},
16031682
{0x6201, 0xE900}, {0x6202, 0xE901}, {0x6203, 0xE902}, {0x6204, 0xE903},
16041683
{0x6205, 0xE904}, {0x6206, 0xE905}, {0x6207, 0xE906}, {0x6208, 0xE907},
16051684
{0x6209, 0xE908}, {0x620A, 0xE909}, {0x620B, 0xE90A}, {0x620C, 0xE90B},
@@ -1935,6 +2014,7 @@ static bool direct_evo_token_for_legacy(uint16_t legacyToken, uint16_t& evoToken
19352014
{0xEF31, 0xE6C5},
19362015
{0x0072, 0xE81B},
19372016
{0x005F, 0xE81C},
2017+
{0xBBEA, 0xF012},
19382018
};
19392019

19402020
const auto it = direct.find(legacyToken);
@@ -1948,20 +2028,8 @@ static bool direct_evo_token_for_legacy(uint16_t legacyToken, uint16_t& evoToken
19482028

19492029
static bool tokenized_legacy_payload_for_evo(uint16_t evoToken, data_t& payload)
19502030
{
1951-
if (legacy_payload_for_evo_ucs2(evoToken, payload))
1952-
{
1953-
return true;
1954-
}
1955-
1956-
uint16_t legacyToken = 0;
1957-
if (!direct_legacy_token_for_evo(evoToken, legacyToken))
1958-
{
1959-
return false;
1960-
}
1961-
1962-
payload.clear();
1963-
append_legacy_token(payload, legacyToken);
1964-
return true;
2031+
return direct_legacy_payload_for_evo(evoToken, payload)
2032+
|| legacy_payload_for_evo_ucs2(evoToken, payload);
19652033
}
19662034

19672035
data_t evo_tokenized_data_to_legacy(const data_t& evoData)

‎tests.cpp‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include <array>
1717
#include <vector>
1818
#include <tuple>
19+
#include <utility>
1920

2021
#ifndef _WIN32
2122
#include <sys/stat.h>
@@ -64,6 +65,26 @@ struct ScopedStderrCapture
6465
}
6566
};
6667

68+
static std::string utf8_from_bmp_codepoint(uint16_t codepoint)
69+
{
70+
if (codepoint < 0x80)
71+
{
72+
return std::string(1, static_cast<char>(codepoint));
73+
}
74+
if (codepoint < 0x800)
75+
{
76+
return {
77+
static_cast<char>(0xC0 | (codepoint >> 6)),
78+
static_cast<char>(0x80 | (codepoint & 0x3F)),
79+
};
80+
}
81+
return {
82+
static_cast<char>(0xE0 | (codepoint >> 12)),
83+
static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F)),
84+
static_cast<char>(0x80 | (codepoint & 0x3F)),
85+
};
86+
}
87+
6788
static void assert_roundtrip_from_readable(TIVarFile& original, const options_t& readableOptions = options_t{})
6889
{
6990
assert(original.hasMultipleEntries() == false);
@@ -1614,6 +1635,51 @@ int main(int argc, char** argv)
16141635
assert(checkedAliasTokenCount == 544);
16151636
}
16161637

1638+
{
1639+
const auto evo_token_data = [](uint16_t token) {
1640+
return data_t{
1641+
static_cast<uint8_t>(token & 0xFF),
1642+
static_cast<uint8_t>((token >> 8) & 0xFF),
1643+
0x00,
1644+
0x00
1645+
};
1646+
};
1647+
1648+
const std::vector<std::pair<uint16_t, data_t>> evoToLegacyAliases = {
1649+
{0xE6AE, {0x02, 0x00, 0xEF, 0x79}},
1650+
{0xE93B, {0x02, 0x00, 0x62, 0x03}},
1651+
{0xE93C, {0x02, 0x00, 0x62, 0x2B}},
1652+
{0xE93D, {0x02, 0x00, 0x62, 0x2E}},
1653+
{0xE93E, {0x02, 0x00, 0xBB, 0xA6}},
1654+
{0xE941, {0x02, 0x00, 0x62, 0x2D}},
1655+
{0xE942, {0x02, 0x00, 0x62, 0x30}},
1656+
{0xE943, {0x02, 0x00, 0x62, 0x06}},
1657+
{0xE944, {0x02, 0x00, 0x62, 0x2C}},
1658+
{0xE945, {0x02, 0x00, 0x62, 0x2F}},
1659+
{0xE946, {0x02, 0x00, 0xBB, 0xCB}},
1660+
{0xE95C, {0x02, 0x00, 0x62, 0x27}},
1661+
{0xE9D6, {0x03, 0x00, 0xBB, 0xEC, 0x0A}},
1662+
{0xE9D7, {0x03, 0x00, 0xBB, 0xEC, 0xAF}},
1663+
{0xE9D8, {0x03, 0x00, 0xBB, 0xEC, 0x0B}},
1664+
};
1665+
1666+
for (const auto& [evoToken, expectedLegacyData] : evoToLegacyAliases)
1667+
{
1668+
ScopedStderrCapture stderrCapture;
1669+
assert(EvoFormat::evo_tokenized_data_to_legacy(evo_token_data(evoToken)) == expectedLegacyData);
1670+
assert(stderrCapture.str().find("Cannot convert Evo token") == std::string::npos);
1671+
}
1672+
1673+
assert(EvoFormat::detokenize_evo_token_words(evo_token_data(0xE9D6)) == "►ʳ");
1674+
assert(EvoFormat::detokenize_evo_token_words(evo_token_data(0xE9D7)) == "►ᵍ");
1675+
assert(EvoFormat::detokenize_evo_token_words(evo_token_data(0xE9D8)) == "►º");
1676+
assert(EvoFormat::detokenize_evo_token_words(evo_token_data(0xF012)) == "₁₀");
1677+
assert(EvoFormat::detokenize_evo_token_words(evo_token_data(0xF029)) == "░");
1678+
1679+
assert(EvoFormat::legacy_tokenized_data_to_evo({0x02, 0x00, 0xEF, 0x79}) == evo_token_data(0xE6AE));
1680+
assert(EvoFormat::legacy_tokenized_data_to_evo({0x02, 0x00, 0xBB, 0xEA}) == evo_token_data(0xF012));
1681+
}
1682+
16171683
{
16181684
const std::string fromGradPath = "/tmp/tivars_evo_from_grad.8xp2";
16191685
TIVarFile fromGradProgram = TIVarFile::createNew("Program", "GRAD", "84Evo");
@@ -1694,6 +1760,56 @@ int main(int argc, char** argv)
16941760
assert(code.find("\\u0178") != std::string::npos);
16951761
}
16961762

1763+
{
1764+
const std::vector<std::pair<uint16_t, uint16_t>> acceptedUcs2Ranges = {
1765+
{0x0394, 0x0394}, {0x03A3, 0x03A3}, {0x03A9, 0x03A9},
1766+
{0x03B1, 0x03B5}, {0x03B8, 0x03B8}, {0x03BB, 0x03BC},
1767+
{0x03C0, 0x03C1}, {0x03C3, 0x03C4}, {0x03C6, 0x03C7},
1768+
{0x2010, 0x2010}, {0x2026, 0x2026}, {0x2070, 0x2070},
1769+
{0x2074, 0x2079}, {0x2080, 0x2089}, {0x2122, 0x2122},
1770+
{0x2190, 0x2193}, {0x221A, 0x221A}, {0x2220, 0x2220},
1771+
{0x222B, 0x222B}, {0x2260, 0x2260}, {0x2264, 0x2265},
1772+
{0x238C, 0x238C}, {0x25A0, 0x25A0}, {0x25AB, 0x25AB},
1773+
{0x25B2, 0x25B2}, {0x25B6, 0x25B6}, {0x25B8, 0x25B8},
1774+
{0x25BC, 0x25BC}, {0x25C0, 0x25C0}, {0x25C2, 0x25C2},
1775+
{0xF000, 0xF032}, {0xF038, 0xF03A}, {0xF041, 0xF04D},
1776+
{0xF04F, 0xF058}, {0xF05B, 0xF061},
1777+
};
1778+
1779+
data_t rawData;
1780+
std::string expectedCode;
1781+
for (const auto& [first, last] : acceptedUcs2Ranges)
1782+
{
1783+
for (uint16_t codepoint = first; codepoint <= last; ++codepoint)
1784+
{
1785+
rawData.push_back(static_cast<uint8_t>(codepoint & 0xFF));
1786+
rawData.push_back(static_cast<uint8_t>((codepoint >> 8) & 0xFF));
1787+
expectedCode += EvoFormat::detokenize_evo_token_words({
1788+
static_cast<uint8_t>(codepoint & 0xFF),
1789+
static_cast<uint8_t>((codepoint >> 8) & 0xFF)
1790+
});
1791+
}
1792+
}
1793+
rawData.push_back(0x00);
1794+
rawData.push_back(0x00);
1795+
1796+
TIVarFile rawAcceptedUcs2Program = TIVarFile::createNew("Program", "UCS2OK", "84Evo");
1797+
rawAcceptedUcs2Program.setContentFromData(rawData);
1798+
assert(json::parse(rawAcceptedUcs2Program.getReadableContent())["code"] == expectedCode);
1799+
1800+
assert(EvoFormat::tokenize_evo_token_words(utf8_from_bmp_codepoint(0x25C2)) == data_t({0xC2, 0x25, 0x00, 0x00}));
1801+
assert(EvoFormat::tokenize_evo_token_words(utf8_from_bmp_codepoint(0xF061)) == data_t({0x61, 0xF0, 0x00, 0x00}));
1802+
assert(EvoFormat::tokenize_evo_token_words("\\u03C0") == data_t({0xC0, 0x03, 0x00, 0x00}));
1803+
assert(EvoFormat::tokenize_evo_token_words("\\uF000") == data_t({0x00, 0xF0, 0x00, 0x00}));
1804+
assert(EvoFormat::evo_tokenized_data_to_legacy({0xC0, 0x03, 0x00, 0x00}) == data_t({0x02, 0x00, 0xBB, 0xA7}));
1805+
1806+
TIVarFile rawRejectedUcs2Program = TIVarFile::createNew("Program", "UCS2BAD", "84Evo");
1807+
rawRejectedUcs2Program.setContentFromData({0xFF, 0x02, 0x10, 0x20, 0x00, 0x00});
1808+
const std::string rejectedCode = json::parse(rawRejectedUcs2Program.getReadableContent())["code"];
1809+
assert(rejectedCode.find("\\u02FF") != std::string::npos);
1810+
assert(rejectedCode.find("\\u2010") == std::string::npos);
1811+
}
1812+
16971813
{
16981814
assert(EvoFormat::tokenize_evo_token_words("ŷ") == data_t({0x77, 0x01, 0x00, 0x00}));
16991815

0 commit comments

Comments
 (0)