Evo: handle even more UCS-2 ranges and direct token mappings.

Credits to Zeroko mostly for painfully testing ranges.

Author: adriweb

src/EvoFormat.cpp

@@ -15,6 +15,7 @@
 #include <sstream>
 #include <stdexcept>
 #include <unordered_map>
+#include <utility>
 
 #include "TIVarTypes.h"
 #include "TypeHandlers/TypeHandlers.h"
@@ -58,9 +59,24 @@ namespace tivars::EvoFormat
 
         bool is_displayable_ucs2_scalar(uint16_t codepoint)
         {
-            return (codepoint >= 0x0020 && codepoint <= 0x007E)
-                || (codepoint >= 0x00A0 && codepoint <= 0x00FF)
-                || codepoint == 0x0177;
+            static constexpr std::pair<uint16_t, uint16_t> acceptedRanges[] = {
+                {0x0020, 0x007E}, {0x00A0, 0x00FF}, {0x0177, 0x0177}, {0x0394, 0x0394},
+                {0x03A3, 0x03A3}, {0x03A9, 0x03A9}, {0x03B1, 0x03B5}, {0x03B8, 0x03B8},
+                {0x03BB, 0x03BC}, {0x03C0, 0x03C1}, {0x03C3, 0x03C4}, {0x03C6, 0x03C7},
+                {0x2010, 0x2010}, {0x2026, 0x2026}, {0x2070, 0x2070}, {0x2074, 0x2079},
+                {0x2080, 0x2089}, {0x2122, 0x2122}, {0x2190, 0x2193}, {0x221A, 0x221A},
+                {0x2220, 0x2220}, {0x222B, 0x222B}, {0x2260, 0x2260}, {0x2264, 0x2265},
+                {0x238C, 0x238C}, {0x25A0, 0x25A0}, {0x25AB, 0x25AB}, {0x25B2, 0x25B2},
+                {0x25B6, 0x25B6}, {0x25B8, 0x25B8}, {0x25BC, 0x25BC}, {0x25C0, 0x25C0},
+                {0x25C2, 0x25C2}, {0xF000, 0xF032}, {0xF038, 0xF03A}, {0xF041, 0xF04D},
+                {0xF04F, 0xF058}, {0xF05B, 0xF061},
+            };
+
+            return std::ranges::any_of(acceptedRanges, [codepoint](const auto& range)
+            {
+                const auto& [first, last] = range;
+                return codepoint >= first && codepoint <= last;
+            });
         }
 
         bool utf8_to_single_codepoint(const std::string& text, uint16_t& codepoint)
@@ -752,6 +768,7 @@ static const char* evo_token_name(uint16_t token)
 }
 
 static bool direct_legacy_token_for_evo(uint16_t evoToken, uint16_t& legacyToken);
+static bool direct_legacy_payload_for_evo(uint16_t evoToken, data_t& payload);
 static bool direct_evo_token_for_legacy(uint16_t legacyToken, uint16_t& evoToken);
 static void append_evo_token(data_t& out, uint16_t evoToken);
 static bool legacy_token_to_evo_ucs2(uint16_t legacyToken, uint16_t& evoToken);
@@ -770,6 +787,9 @@ static std::string evo_token_to_string(uint16_t token)
     if (token == 0xE41A) return "'";
     if (token == 0xE424) return "ᵍ";
     if (token == 0xE589) return "Grad";
+    if (token == 0xE9D6) return "►ʳ";
+    if (token == 0xE9D7) return "►ᵍ";
+    if (token == 0xE9D8) return "►º";
     if (token >= 0xE850 && token <= 0xE85B)
     {
         const uint16_t idx = static_cast<uint16_t>((token - 0xE850) / 2 + 1);
@@ -946,6 +966,28 @@ static bool legacy_payload_for_evo_ucs2(uint16_t evoToken, data_t& payload)
     }
 }
 
+static bool direct_legacy_payload_for_evo(uint16_t evoToken, data_t& payload)
+{
+    payload.clear();
+
+    // ►{angle} conv token
+    if (evoToken == 0xE9D6 || evoToken == 0xE9D7 || evoToken == 0xE9D8)
+    {
+        append_legacy_token(payload, 0xBBEC);
+        append_legacy_token(payload, evoToken == 0xE9D6 ? 0x0A : evoToken == 0xE9D7 ? 0xAF : 0x0B);
+        return true;
+    }
+
+    uint16_t legacyToken = 0;
+    if (!direct_legacy_token_for_evo(evoToken, legacyToken))
+    {
+        return false;
+    }
+
+    append_legacy_token(payload, legacyToken);
+    return true;
+}
+
 static bool legacy_token_to_evo_ucs2(uint16_t legacyToken, uint16_t& evoToken)
 {
     const std::string text = TypeHandlers::TH_Tokenized::oneTokenBytesToString(legacyToken);
@@ -1105,6 +1147,7 @@ static bool direct_legacy_token_for_evo(uint16_t evoToken, uint16_t& legacyToken
         {0xE5BD, 0x7F}, {0xE5BE, 0x80}, {0xE5BF, 0x81}, {0xE5C0, 0xEF73},
         {0xE5C1, 0xEF74}, {0xE5C2, 0xEF75},
         {0xE4F9, 0xBB57}, {0xE593, 0xBB64}, {0xE6C6, 0xE8}, {0xE6C7, 0xE7},
+        {0xE6AE, 0xEF79},
         {0xE900, 0x6201}, {0xE901, 0x6202}, {0xE902, 0x6203}, {0xE903, 0x6204},
         {0xE904, 0x6205}, {0xE905, 0x6206}, {0xE906, 0x6207}, {0xE907, 0x6208},
         {0xE908, 0x6209}, {0xE909, 0x620A}, {0xE90A, 0x620B}, {0xE90B, 0x620C},
@@ -1121,6 +1164,9 @@ static bool direct_legacy_token_for_evo(uint16_t evoToken, uint16_t& legacyToken
         {0xE932, 0x6234}, {0xE933, 0x6235}, {0xE934, 0x6236}, {0xE935, 0x6237},
         {0xE936, 0x6238}, {0xE937, 0x6239}, {0xE938, 0x623A}, {0xE939, 0x623B},
         {0xE93A, 0x623C}, {0xE980, 0x6304}, {0xE981, 0x6305}, {0xE982, 0x6332},
+        {0xE93B, 0x6203}, {0xE93C, 0x622B}, {0xE93D, 0x622E}, {0xE93E, 0xBBA6},
+        {0xE941, 0x622D}, {0xE942, 0x6230}, {0xE943, 0x6206}, {0xE944, 0x622C},
+        {0xE945, 0x622F}, {0xE946, 0xBBCB}, {0xE95C, 0x6227},
         {0xE983, 0x6306}, {0xE984, 0x6307}, {0xE985, 0x6308}, {0xE986, 0x6309},
         {0xE987, 0x6333}, {0xE98F, 0x630A}, {0xE990, 0x630B}, {0xE991, 0x6302},
         {0xE992, 0x6336}, {0xE993, 0x630C}, {0xE994, 0x630D}, {0xE995, 0x6303},
@@ -1600,6 +1646,7 @@ static bool direct_evo_token_for_legacy(uint16_t legacyToken, uint16_t& evoToken
         {0x7F, 0xE5BD}, {0x80, 0xE5BE}, {0x81, 0xE5BF}, {0xEF73, 0xE5C0},
         {0xEF74, 0xE5C1}, {0xEF75, 0xE5C2},
         {0xBB57, 0xE4F9}, {0xBB64, 0xE593}, {0xE8, 0xE6C6}, {0xE7, 0xE6C7},
+        {0xEF79, 0xE6AE},
         {0x6201, 0xE900}, {0x6202, 0xE901}, {0x6203, 0xE902}, {0x6204, 0xE903},
         {0x6205, 0xE904}, {0x6206, 0xE905}, {0x6207, 0xE906}, {0x6208, 0xE907},
         {0x6209, 0xE908}, {0x620A, 0xE909}, {0x620B, 0xE90A}, {0x620C, 0xE90B},
@@ -1948,20 +1995,8 @@ static bool direct_evo_token_for_legacy(uint16_t legacyToken, uint16_t& evoToken
 
 static bool tokenized_legacy_payload_for_evo(uint16_t evoToken, data_t& payload)
 {
-    if (legacy_payload_for_evo_ucs2(evoToken, payload))
-    {
-        return true;
-    }
-
-    uint16_t legacyToken = 0;
-    if (!direct_legacy_token_for_evo(evoToken, legacyToken))
-    {
-        return false;
-    }
-
-    payload.clear();
-    append_legacy_token(payload, legacyToken);
-    return true;
+    return direct_legacy_payload_for_evo(evoToken, payload)
+        || legacy_payload_for_evo_ucs2(evoToken, payload);
 }
 
 data_t evo_tokenized_data_to_legacy(const data_t& evoData)

tests.cpp

@@ -16,6 +16,7 @@
 #include <array>
 #include <vector>
 #include <tuple>
+#include <utility>
 
 #ifndef _WIN32
     #include <sys/stat.h>
@@ -64,6 +65,26 @@ struct ScopedStderrCapture
     }
 };
 
+static std::string utf8_from_bmp_codepoint(uint16_t codepoint)
+{
+    if (codepoint < 0x80)
+    {
+        return std::string(1, static_cast<char>(codepoint));
+    }
+    if (codepoint < 0x800)
+    {
+        return {
+            static_cast<char>(0xC0 | (codepoint >> 6)),
+            static_cast<char>(0x80 | (codepoint & 0x3F)),
+        };
+    }
+    return {
+        static_cast<char>(0xE0 | (codepoint >> 12)),
+        static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F)),
+        static_cast<char>(0x80 | (codepoint & 0x3F)),
+    };
+}
+
 static void assert_roundtrip_from_readable(TIVarFile& original, const options_t& readableOptions = options_t{})
 {
     assert(original.hasMultipleEntries() == false);
@@ -1614,6 +1635,48 @@ int main(int argc, char** argv)
         assert(checkedAliasTokenCount == 544);
     }
 
+    {
+        const auto evo_token_data = [](uint16_t token) {
+            return data_t{
+                static_cast<uint8_t>(token & 0xFF),
+                static_cast<uint8_t>((token >> 8) & 0xFF),
+                0x00,
+                0x00
+            };
+        };
+
+        const std::vector<std::pair<uint16_t, data_t>> evoToLegacyAliases = {
+            {0xE6AE, {0x02, 0x00, 0xEF, 0x79}},
+            {0xE93B, {0x02, 0x00, 0x62, 0x03}},
+            {0xE93C, {0x02, 0x00, 0x62, 0x2B}},
+            {0xE93D, {0x02, 0x00, 0x62, 0x2E}},
+            {0xE93E, {0x02, 0x00, 0xBB, 0xA6}},
+            {0xE941, {0x02, 0x00, 0x62, 0x2D}},
+            {0xE942, {0x02, 0x00, 0x62, 0x30}},
+            {0xE943, {0x02, 0x00, 0x62, 0x06}},
+            {0xE944, {0x02, 0x00, 0x62, 0x2C}},
+            {0xE945, {0x02, 0x00, 0x62, 0x2F}},
+            {0xE946, {0x02, 0x00, 0xBB, 0xCB}},
+            {0xE95C, {0x02, 0x00, 0x62, 0x27}},
+            {0xE9D6, {0x03, 0x00, 0xBB, 0xEC, 0x0A}},
+            {0xE9D7, {0x03, 0x00, 0xBB, 0xEC, 0xAF}},
+            {0xE9D8, {0x03, 0x00, 0xBB, 0xEC, 0x0B}},
+        };
+
+        for (const auto& [evoToken, expectedLegacyData] : evoToLegacyAliases)
+        {
+            ScopedStderrCapture stderrCapture;
+            assert(EvoFormat::evo_tokenized_data_to_legacy(evo_token_data(evoToken)) == expectedLegacyData);
+            assert(stderrCapture.str().find("Cannot convert Evo token") == std::string::npos);
+        }
+
+        assert(EvoFormat::detokenize_evo_token_words(evo_token_data(0xE9D6)) == "►ʳ");
+        assert(EvoFormat::detokenize_evo_token_words(evo_token_data(0xE9D7)) == "►ᵍ");
+        assert(EvoFormat::detokenize_evo_token_words(evo_token_data(0xE9D8)) == "►º");
+
+        assert(EvoFormat::legacy_tokenized_data_to_evo({0x02, 0x00, 0xEF, 0x79}) == evo_token_data(0xE6AE));
+    }
+
     {
         const std::string fromGradPath = "/tmp/tivars_evo_from_grad.8xp2";
         TIVarFile fromGradProgram = TIVarFile::createNew("Program", "GRAD", "84Evo");
@@ -1694,6 +1757,53 @@ int main(int argc, char** argv)
         assert(code.find("\\u0178") != std::string::npos);
     }
 
+    {
+        const std::vector<std::pair<uint16_t, uint16_t>> acceptedUcs2Ranges = {
+            {0x0394, 0x0394}, {0x03A3, 0x03A3}, {0x03A9, 0x03A9},
+            {0x03B1, 0x03B5}, {0x03B8, 0x03B8}, {0x03BB, 0x03BC},
+            {0x03C0, 0x03C1}, {0x03C3, 0x03C4}, {0x03C6, 0x03C7},
+            {0x2010, 0x2010}, {0x2026, 0x2026}, {0x2070, 0x2070},
+            {0x2074, 0x2079}, {0x2080, 0x2089}, {0x2122, 0x2122},
+            {0x2190, 0x2193}, {0x221A, 0x221A}, {0x2220, 0x2220},
+            {0x222B, 0x222B}, {0x2260, 0x2260}, {0x2264, 0x2265},
+            {0x238C, 0x238C}, {0x25A0, 0x25A0}, {0x25AB, 0x25AB},
+            {0x25B2, 0x25B2}, {0x25B6, 0x25B6}, {0x25B8, 0x25B8},
+            {0x25BC, 0x25BC}, {0x25C0, 0x25C0}, {0x25C2, 0x25C2},
+            {0xF000, 0xF032}, {0xF038, 0xF03A}, {0xF041, 0xF04D},
+            {0xF04F, 0xF058}, {0xF05B, 0xF061},
+        };
+
+        data_t rawData;
+        std::string expectedCode;
+        for (const auto& [first, last] : acceptedUcs2Ranges)
+        {
+            for (uint16_t codepoint = first; codepoint <= last; ++codepoint)
+            {
+                rawData.push_back(static_cast<uint8_t>(codepoint & 0xFF));
+                rawData.push_back(static_cast<uint8_t>((codepoint >> 8) & 0xFF));
+                expectedCode += utf8_from_bmp_codepoint(codepoint);
+            }
+        }
+        rawData.push_back(0x00);
+        rawData.push_back(0x00);
+
+        TIVarFile rawAcceptedUcs2Program = TIVarFile::createNew("Program", "UCS2OK", "84Evo");
+        rawAcceptedUcs2Program.setContentFromData(rawData);
+        assert(json::parse(rawAcceptedUcs2Program.getReadableContent())["code"] == expectedCode);
+
+        assert(EvoFormat::tokenize_evo_token_words(utf8_from_bmp_codepoint(0x25C2)) == data_t({0xC2, 0x25, 0x00, 0x00}));
+        assert(EvoFormat::tokenize_evo_token_words(utf8_from_bmp_codepoint(0xF061)) == data_t({0x61, 0xF0, 0x00, 0x00}));
+        assert(EvoFormat::tokenize_evo_token_words("\\u03C0") == data_t({0xC0, 0x03, 0x00, 0x00}));
+        assert(EvoFormat::tokenize_evo_token_words("\\uF000") == data_t({0x00, 0xF0, 0x00, 0x00}));
+        assert(EvoFormat::evo_tokenized_data_to_legacy({0xC0, 0x03, 0x00, 0x00}) == data_t({0x02, 0x00, 0xBB, 0xA7}));
+
+        TIVarFile rawRejectedUcs2Program = TIVarFile::createNew("Program", "UCS2BAD", "84Evo");
+        rawRejectedUcs2Program.setContentFromData({0xFF, 0x02, 0x10, 0x20, 0x00, 0x00});
+        const std::string rejectedCode = json::parse(rawRejectedUcs2Program.getReadableContent())["code"];
+        assert(rejectedCode.find("\\u02FF") != std::string::npos);
+        assert(rejectedCode.find("\\u2010") == std::string::npos);
+    }
+
     {
         assert(EvoFormat::tokenize_evo_token_words("ŷ") == data_t({0x77, 0x01, 0x00, 0x00}));