add support for ANSI files containing non-ASCII

Notepad++ internally represents files with most encodings as UTF8,
	but ANSI files are internally represented as ANSI.
Previously JsonTools assumed that any method
	that got the bytes of the document from Notepad++
	could treat those bytes as UTF8,
	and that assumption is wrong for ANSI.
Now JsonTools uses ScintillaGateway.GetCodePage()
	to determine the encoding of the document and requires
	an encoding as a parameter for many document-text-extracting functions.

However, the JsonParser class is still hardcoded
	to count the bytes in the UTF8 representation of each character in a string,
	and it would be both annoying and potentially performance-reducing to stop making this assumption.
In a future commit I may address this issue, but for now some features are broken for non-ASCII ANSI files, including the following:
	- automagic navigation to position in document of a tree node
	- selecting a tree node or its children
	- getting the path to a position in the document
	- selecting all valid JSON in a selection

Author: molsonkiko

CHANGELOG.md

@@ -54,8 +54,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
-1. Bug where JsonTools could not set the text of a file to end with the SOH character (ASCII code `\x01`)
-2. Bug where some JsonGrepper tests would always fail because they queried an external API that had previously returned JSON but now returns HTML.
+1. __Correctly parse all ANSI files.__ Previously JsonTools assumed that Notepad++ used UTF8 to internally represent every document, and while that appears to have worked fine for most encodings, it would specifically fail on ANSI-encoded documents containing non-ASCII.
+    - __The following features are currently *still broken* on ANSI files with non-ASCII characters__ (because they rely on knowing the position of a JSON node):
+		* [path to current position](/docs/README.md#path-to-current-position)
+		* [clicking on a tree node to navigate to its position in the document](/docs/README.md#the-basics)
+		* [selecting a tree node or its children](/docs/README.md#select-tree-nodes-json-or-its-children-added-in-v57)
+		* [selecting all valid JSON in selection](/docs/README.md#selecting-all-valid-json)
+2. Minor bug where JsonTools could not set the text of a file to end with the SOH character (ASCII code `\x01`)
+3. Minor bug where some JsonGrepper tests would always fail because they queried an external API that had previously returned JSON but now returns HTML.
 
 ## [8.4.0] - 2025-05-04
 

JsonToolsNppPlugin/Forms/TreeViewer.cs

@@ -1248,6 +1248,7 @@ public static bool LengthOfStringInRegexMode(JNode[] nodes, char delim, char quo
             int startIndex = 0;
             string documentText = null;
             utf8Lengths = new int[nodes.Length];
+            Encoding encoding = Npp.editor.GetCodePage();
             for (int ii = 0; ii < nodes.Length; ii++)
             {
                 JNode jnode = nodes[ii];
@@ -1269,7 +1270,7 @@ public static bool LengthOfStringInRegexMode(JNode[] nodes, char delim, char quo
                     if (documentText is null)
                         documentText = selectionEnd < 0
                             ? Npp.editor.GetText()
-                            : Npp.GetSlice(selectionStart, selectionEnd);
+                            : Npp.GetSlice(selectionStart, selectionEnd, encoding);
                     int utf8Extra = 0;
                     for (; startIndex < documentText.Length && startIndex + utf8Extra < nodepos; startIndex++)
                         utf8Extra += JsonParser.ExtraUTF8Bytes(documentText[startIndex]);
@@ -1298,7 +1299,7 @@ public static bool LengthOfStringInRegexMode(JNode[] nodes, char delim, char quo
                         // even if a string doesn't *need* to be quoted, it could be quoted anyway, in which case we need to select the quotes
                         int nodeStart = selectionStart + jnode.position;
                         int quoteLen = 1 + JsonParser.ExtraUTF8Bytes(quote);
-                        string firstChar = Npp.GetSlice(nodeStart, nodeStart + quoteLen);
+                        string firstChar = Npp.GetSlice(nodeStart, nodeStart + quoteLen, encoding);
                         if (firstChar[0] == quote)
                             utf8Len += 2 * quoteLen;
                     }

JsonToolsNppPlugin/Main.cs

@@ -652,9 +652,10 @@ public static (ParserState parserState, JNode node, bool usesSelections, Documen
                 if (wasAutotriggered && combinedLength > sizeThreshold)
                     return (ParserState.OK, null, false, DocumentType.NONE);
                 stopUsingSelections = false;
+                Encoding encoding = Npp.editor.GetCodePage();
                 foreach ((int start, int end) in selRanges)
                 {
-                    string selRange = Npp.GetSlice(start, end);
+                    string selRange = Npp.GetSlice(start, end, encoding);
                     JNode subJson;
                     if (documentType == DocumentType.REGEX)
                     {
@@ -959,6 +960,7 @@ public static void CompressJson()
             Npp.editor.BeginUndoAction();
             if (usesSelections)
             {
+                Encoding encoding = Npp.editor.GetCodePage();
                 var obj = (JObject)json;
                 int delta = 0;
                 pluginIsEditing = true;
@@ -976,7 +978,7 @@ public static void CompressJson()
                     int newStart = start + delta;
                     int newCount = Encoding.UTF8.GetByteCount(printed);
                     Npp.editor.DeleteRange(newStart, oldLen);
-                    Npp.editor.InsertText(newStart, printed);
+                    Npp.editor.InsertText(newStart, printed, encoding);
                     currentIndicator = ApplyAndSwapIndicator(currentIndicator, newStart, newCount);
                     int newEnd = newStart + newCount;
                     delta = newEnd - end;
@@ -1071,9 +1073,10 @@ public static void DumpSelectedTextAsJsonString()
             else
             {
                 var sb = new StringBuilder();
+                Encoding encoding = Npp.editor.GetCodePage();
                 foreach ((int start, int end) in selections)
                 {
-                    string sel = Npp.GetSlice(start, end);
+                    string sel = Npp.GetSlice(start, end, encoding);
                     JNode selNode = new JNode(sel);
                     sb.Append(selNode.ToString());
                     sb.Append("\r\n");
@@ -1096,10 +1099,11 @@ public static void DumpSelectedJsonStringsAsText()
             var selections = SelectionManager.GetSelectedRanges();
             selections.Sort(SelectionManager.StartEndCompareByStart);
             var sb = new StringBuilder();
+            Encoding encoding = Npp.editor.GetCodePage();
             JsonParser jsonParser = JsonParserFromSettings();
             if (SelectionManager.NoTextSelected(selections))
             {
-                string selStrValue = TryGetSelectedJsonStringValue(jsonParser);
+                string selStrValue = TryGetSelectedJsonStringValue(jsonParser, encoding);
                 if (selStrValue == null)
                     return;
                 sb.Append(selStrValue);
@@ -1108,7 +1112,7 @@ public static void DumpSelectedJsonStringsAsText()
             {
                 foreach ((int start, int end) in selections)
                 {
-                    string selStrValue = TryGetSelectedJsonStringValue(jsonParser, start, end);
+                    string selStrValue = TryGetSelectedJsonStringValue(jsonParser, encoding, start, end);
                     if (selStrValue == null)
                         return;
                     sb.Append(selStrValue);
@@ -1119,11 +1123,11 @@ public static void DumpSelectedJsonStringsAsText()
             Npp.editor.SetText(sb.ToString());
         }
 
-        public static string TryGetSelectedJsonStringValue(JsonParser jsonParser, int start = -1, int end = -1)
+        public static string TryGetSelectedJsonStringValue(JsonParser jsonParser, Encoding encoding, int start = -1, int end = -1)
         {
             string text = start < 0 || end < 0
                 ? Npp.editor.GetText()
-                : Npp.GetSlice(start, end);
+                : Npp.GetSlice(start, end, encoding);
             try
             {
                 JNode textNode = jsonParser.Parse(text);
@@ -1454,7 +1458,8 @@ private static string PathToPosition(KeyStyle style, char separator, int pos = -
                     (int start, int end) = SelectionManager.GetEnclosingRememberedSelection(pos, selectionRememberingIndicator1, selectionRememberingIndicator2);
                     if (start < 0)
                         return "";
-                    string selText = Npp.GetSlice(start, end);
+                    var encoding = Npp.editor.GetCodePage();
+                    string selText = Npp.GetSlice(start, end, encoding);
                     var parser = JsonParserFromSettings();
                     JNode selJson = parser.Parse(selText);
                     if (parser.fatal)
@@ -1496,9 +1501,10 @@ public static void SelectEveryValidJson()
             var startEnds = new List<string>();
             int lastEnd = 0;
             Predicate<char> isTryParseStart = c => settings.try_parse_start_chars.IndexOf(c) >= 0;
+            var encoding = Npp.editor.GetCodePage();
             foreach ((int start, int end) in selections)
             {
-                string text = Npp.GetSlice(start, end);
+                string text = Npp.GetSlice(start, end, encoding);
                 int ii = 0;
                 int len = text.Length;
                 int utf8Pos = start;
@@ -2016,7 +2022,7 @@ public static void OpenSortForm()
         /// <returns></returns>
         public static int EndOfJNodeAtPos(int startUtf8Pos, int end)
         {
-            string slice = Npp.GetSlice(startUtf8Pos, end);
+            string slice = Npp.GetSlice(startUtf8Pos, end, Npp.editor.GetCodePage());
             var parser = new JsonParser(LoggerLevel.JSON5);
             try
             {
@@ -2055,7 +2061,8 @@ public static void SelectAllChildren(IEnumerable<int> positions, bool isJsonLine
                     "Can't select all children", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
                 return;
             }
-            string slice = Npp.GetSlice(minPos, len);
+            Encoding encoding = Npp.editor.GetCodePage();
+            string slice = Npp.GetSlice(minPos, len,  encoding);
             var parser = new JsonParser(LoggerLevel.JSON5);
             int utf8ExtraBytes = 0;
             int positionsIdx = 0;

JsonToolsNppPlugin/PluginInfrastructure/IScintillaGateway.cs

@@ -1,5 +1,6 @@
 // NPP plugin platform for .Net v0.94.00 by Kasper B. Graversen etc.
 using System;
+using System.Text;
 using static Kbg.NppPluginNET.PluginInfrastructure.Win32;
 
 namespace Kbg.NppPluginNET.PluginInfrastructure
@@ -15,7 +16,7 @@ public interface IScintillaGateway
     {
         int GetSelectionLength();
         void AppendTextAndMoveCursor(string text);
-        void InsertTextAndMoveCursor(string text);
+        void InsertTextAndMoveCursor(string text, Encoding encoding);
         void SelectCurrentLine();
         void ClearSelectionToCursor();
 
@@ -40,7 +41,7 @@ public interface IScintillaGateway
         unsafe void AddStyledText(int length, Cells c);
 
         /// <summary>Insert string at a position. (Scintilla feature 2003)</summary>
-        unsafe void InsertText(int pos, string text);
+        unsafe void InsertText(int pos, string text, Encoding encoding);
 
         /// <summary>Change the text that is being inserted in response to SC_MOD_INSERTCHECK (Scintilla feature 2672)</summary>
         unsafe void ChangeInsertion(int length, string text);
@@ -761,7 +762,7 @@ public interface IScintillaGateway
         int GetLineEndPosition(int line);
 
         /// <summary>Get the code page used to interpret the bytes of the document as characters. (Scintilla feature 2137)</summary>
-        int GetCodePage();
+        Encoding GetCodePage();
 
         /// <summary>Get the foreground colour of the caret. (Scintilla feature 2138)</summary>
         Colour GetCaretFore();

JsonToolsNppPlugin/PluginInfrastructure/ScintillaGateway.cs

@@ -25,7 +25,8 @@ public class ScintillaGateway : IScintillaGateway
         /// * if length is 0, return the length and have no side effects<br></br>
         /// * if length is greater than 0, return the length and fill the buffer with length characters<br></br>
         /// This method gets the length if the length is 0, uses the second mode to fill a buffer,<br></br>
-        /// and returns a string of the UTF8-decoded buffer with all trailing '\x00' chars stripped off.
+        /// figures out the document's encoding,<br></br>
+        /// and returns a string of the decoded buffer with all trailing '\x00' chars stripped off.
         /// </summary>
         /// <param name="msg">message to send</param>
         /// <param name="length">number of characters to retrieve (if 0, find out by sending message)</param>
@@ -35,6 +36,7 @@ private unsafe string GetNullStrippedStringFromMessageThatReturnsLength(SciMsg m
             if (length < 1)
                 length = Win32.SendMessage(scintilla, msg, (IntPtr)Unused, (IntPtr)Unused).ToInt32();
             byte[] textBuffer = new byte[length];
+            var encoding = GetCodePage();
             fixed (byte* textPtr = textBuffer)
             {
                 Win32.SendMessage(scintilla, msg, (IntPtr)length, (IntPtr)textPtr);
@@ -43,21 +45,21 @@ private unsafe string GetNullStrippedStringFromMessageThatReturnsLength(SciMsg m
                 // other than NULL can have any 0-valued bytes in UTF-8.
                 // See https://en.wikipedia.org/wiki/UTF-8#Encoding
                 for (; lastNullCharPos >= 0 && textBuffer[lastNullCharPos] == '\x00'; lastNullCharPos--) { }
-                return Encoding.UTF8.GetString(textBuffer, 0, lastNullCharPos + 1);
+                return encoding.GetString(textBuffer, 0, lastNullCharPos + 1);
             }
         }
 
         /// <summary>
-        /// the same byte[] buffer that would be returned by Encoding.UTF8.GetBytes(text),
+        /// the same byte[] buffer that would be returned by encoding.GetBytes(text),<br></br>
         /// but with +1 length and a NULL byte at the end
         /// </summary>
         /// <param name="text"></param>
         /// <returns></returns>
-        private byte[] GetNullTerminatedUTF8Bytes(string text)
+        private byte[] GetNullTerminatedEncodedBytes(string text, Encoding encoding)
         {
-            int length = Encoding.UTF8.GetByteCount(text);
+            int length = encoding.GetByteCount(text);
             byte[] bytes = new byte[length + 1];
-            int lengthWritten = Encoding.UTF8.GetBytes(text, 0, text.Length, bytes, 0);
+            int lengthWritten = encoding.GetBytes(text, 0, text.Length, bytes, 0);
             //if (lengthWritten != length)
             //    throw new Exception("not sure what we would do here");
             return bytes;
@@ -80,10 +82,10 @@ public void AppendTextAndMoveCursor(string text)
             GotoPos(GetCurrentPos() + text.Length);
         }
 
-        public void InsertTextAndMoveCursor(string text)
+        public void InsertTextAndMoveCursor(string text, Encoding encoding)
         {
             var currentPos = GetCurrentPos();
-            InsertText(currentPos, text);
+            InsertText(currentPos, text, encoding);
             GotoPos(currentPos + text.Length);
         }
 
@@ -145,9 +147,9 @@ public unsafe void AddStyledText(int length, Cells c)
         }
 
         /// <summary>Insert string at a position. (Scintilla feature 2003)</summary>
-        public unsafe void InsertText(int pos, string text)
+        public unsafe void InsertText(int pos, string text, Encoding encoding)
         {
-            fixed (byte* textPtr = GetNullTerminatedUTF8Bytes(text))
+            fixed (byte* textPtr = GetNullTerminatedEncodedBytes(text, encoding))
             {
                 Win32.SendMessage(scintilla, SciMsg.SCI_INSERTTEXT, (IntPtr) pos, (IntPtr) textPtr);
             }
@@ -1505,9 +1507,17 @@ public int GetLineEndPosition(int line)
         }
 
         /// <summary>Get the code page used to interpret the bytes of the document as characters. (Scintilla feature 2137)</summary>
-        public int GetCodePage()
+        public Encoding GetCodePage()
         {
-            return (int)Win32.SendMessage(scintilla, SciMsg.SCI_GETCODEPAGE, (IntPtr) Unused, (IntPtr) Unused);
+            var cpNum = (int)Win32.SendMessage(scintilla, SciMsg.SCI_GETCODEPAGE, (IntPtr) Unused, (IntPtr) Unused);
+            try
+            {
+                return Encoding.GetEncoding(cpNum);
+            }
+            catch
+            {
+                return Encoding.UTF8;
+            }
         }
 
         /// <summary>Get the foreground colour of the caret. (Scintilla feature 2138)</summary>
@@ -1797,7 +1807,8 @@ public void Clear()
         /// <summary>Replace the contents of the document with the argument text. (Scintilla feature 2181)</summary>
         public unsafe void SetText(string text)
         {
-            fixed (byte* textPtr = GetNullTerminatedUTF8Bytes(text))
+            Encoding encoding = GetCodePage();
+            fixed (byte* textPtr = GetNullTerminatedEncodedBytes(text, encoding))
             {
                 Win32.SendMessage(scintilla, SciMsg.SCI_SETTEXT, (IntPtr) Unused, (IntPtr) textPtr);
             }

JsonToolsNppPlugin/Properties/AssemblyInfo.cs

@@ -28,5 +28,5 @@
 //      Build Number
 //      Revision
 //
-[assembly: AssemblyVersion("8.4.0.1")]
-[assembly: AssemblyFileVersion("8.4.0.1")]
+[assembly: AssemblyVersion("8.4.0.2")]
+[assembly: AssemblyFileVersion("8.4.0.2")]

JsonToolsNppPlugin/Tests/TestRunner.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Text;
 using System.Threading.Tasks;
 using JSON_Tools.Utils;
 using Kbg.NppPluginNET;
@@ -198,10 +199,10 @@ public static async Task RunAll()
                         failures.Add(name);
                 }
             }
-
+            Encoding encoding = Npp.editor.GetCodePage();
             if (skipped.Count > 0)
-                Npp.editor.InsertText(header.Length + 2, "Tests skipped: " + string.Join(", ", skipped) + "\r\n");
-            Npp.editor.InsertText(header.Length + 2, "Tests failed: " + string.Join(", ", failures) + "\r\n");
+                Npp.editor.InsertText(header.Length + 2, "Tests skipped: " + string.Join(", ", skipped) + "\r\n", encoding);
+            Npp.editor.InsertText(header.Length + 2, "Tests failed: " + string.Join(", ", failures) + "\r\n", encoding);
         }
     }
 }

JsonToolsNppPlugin/Tests/UserInterfaceTests.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using System.Windows.Forms;
 using JSON_Tools.Forms;
 using JSON_Tools.JSON_Tools;
@@ -65,7 +66,8 @@ public static bool ExecuteFileManipulation(string command, List<string> messages
                 var start = (int)args[0];
                 text = (string)args[1];
                 messages.Add($"insert {JNode.StrToString(text, false)} at {start}");
-                Npp.editor.InsertText(start, text);
+                Encoding encoding = Npp.editor.GetCodePage();
+                Npp.editor.InsertText(start, text, encoding);
                 break;
             case "delete_text":
                 start = (int)args[0];

JsonToolsNppPlugin/Utils/Npp.cs

@@ -256,14 +256,14 @@ public static void RemoveTrailingSOH()
         /// <param name="start"></param>
         /// <param name="end"></param>
         /// <returns></returns>
-        public static string GetSlice(int start, int end)
+        public static string GetSlice(int start, int end, Encoding encoding)
         {
             int len = end - start;
             IntPtr rangePtr = editor.GetRangePointer(start, len);
             string ansi = Marshal.PtrToStringAnsi(rangePtr, len);
             // TODO: figure out a way to do this that involves less memcopy for non-ASCII
-            if (ansi.Any(c => c >= 128))
-                return Encoding.UTF8.GetString(Encoding.Default.GetBytes(ansi));
+            if (encoding.CodePage != Encoding.Default.CodePage && ansi.Any(c => c >= 128))
+                return encoding.GetString(Encoding.Default.GetBytes(ansi));
             return ansi;
         }