better evo tok

Author: adriweb

src/EvoFormat.cpp

@@ -18,8 +18,11 @@
 
 #include "TIVarTypes.h"
 #include "TypeHandlers/TypeHandlers.h"
+#include "json.hpp"
 #include "tivarslib_utils.h"
 
+using json = nlohmann::ordered_json;
+
 namespace tivars::EvoFormat
 {
     namespace
@@ -113,6 +116,27 @@ namespace tivars::EvoFormat
             codepoint = static_cast<uint16_t>(value);
             return true;
         }
+
+        data_t hex_string_to_bytes(const std::string& hex, const char* fieldName)
+        {
+            if ((hex.size() % 2) != 0)
+            {
+                throw std::invalid_argument(std::string(fieldName) + " must contain an even number of hex digits");
+            }
+
+            data_t out;
+            out.reserve(hex.size() / 2);
+            for (size_t i = 0; i < hex.size(); i += 2)
+            {
+                if (!std::isxdigit(static_cast<unsigned char>(hex[i])) ||
+                    !std::isxdigit(static_cast<unsigned char>(hex[i + 1])))
+                {
+                    throw std::invalid_argument(std::string(fieldName) + " must be valid hexadecimal");
+                }
+                out.push_back(hexdec(hex.substr(i, 2)));
+            }
+            return out;
+        }
     }
 
 uint16_t evo_checksum(const data_t& body)
@@ -804,10 +828,32 @@ data_t tokenize_evo_token_words(const std::string& source, const options_t& opti
     const bool deindent = options.contains("deindent") && options.at("deindent") == 1;
     const bool detectStrings = !options.contains("detect_strings") || options.at("detect_strings") != 0;
 
+    std::string sourceText = source;
+    const std::string trimmed = trim(sourceText);
+    if (!trimmed.empty() && trimmed.front() == '{')
+    {
+        try
+        {
+            const json j = json::parse(trimmed);
+            if (j.contains("rawDataHex"))
+            {
+                return hex_string_to_bytes(j.at("rawDataHex").get<std::string>(), "rawDataHex");
+            }
+            if (j.contains("code"))
+            {
+                sourceText = j.at("code").get<std::string>();
+            }
+        }
+        catch (const json::exception&)
+        {
+            // Ignore non-JSON input and fall back to regular Evo tokenized parsing.
+        }
+    }
+
     std::string normalizedSource;
     if (deindent)
     {
-        std::istringstream lines{source};
+        std::istringstream lines{sourceText};
         std::string line;
         while (std::getline(lines, line))
         {
@@ -820,7 +866,7 @@ data_t tokenize_evo_token_words(const std::string& source, const options_t& opti
     }
     else
     {
-        normalizedSource = source;
+        normalizedSource = sourceText;
     }
 
     static constexpr uint16_t legacyQuote = 0x2A;

tests.cpp

@@ -1704,6 +1704,10 @@ int main(int argc, char** argv)
         const json readable = json::parse(directYHatProgram.getReadableContent());
         assert(readable["code"] == "Disp \"ŷ\"");
         assert(readable["rawDataHex"].get<std::string>().find("7701") != std::string::npos);
+
+        TIVarFile recreatedYHatProgram = TIVarFile::createNew("Program", "YHAT3", "84Evo");
+        recreatedYHatProgram.setContentFromString(directYHatProgram.getReadableContent());
+        assert(recreatedYHatProgram.getRawContent() == directYHatProgram.getRawContent());
     }
 
     {