change RichMultiString.cs to store RichStrings instead of just strings (#1692)

* change RichMultiString.cs to store RichStrings instead of just strings

* migrate IRichMultiString to use IRichString, make a rich multi ws input editor based off lcm-rich-text-editor.svelte

* only fire onchange on blur and not on keystroke

* Add test for property changes in RichMultiString diffs

This test ensures that the diff logic correctly captures property changes, such as toggling text formatting attributes in RichMultiStrings. It validates the generation of precise replace operations when properties are updated.

* don't allow RichSpan.Ws to be null

* handle null strings better, ensure ws it not null where possible.

Note, in the case of ExampleSentence.Reference we don't have a writing system to fallback to like we do with MultiString, so it may be null in some old data, however, it shouldn't trigger any issues due to sync order.

* handle db serialization issues due to ExampleSentence.Reference being a RichString

* add migration for Reference column

* Refactor activity view to use runes

* ensure a default WsId has the Code "default" rather than null

* add ExampleSentenceTestsBase.cs to catch issues creating examples without references

* change RichSpan.Colors to use the C# Color struct to avoid issues of invalid colors

* globally configure FluentAssert to compare RichString and RichSpan by members and no their equals method to improve assert messages

* try to restore cursor location/selection on rich text state change, this avoids issues when typing in a field which was undefined

* ensure all tests projects use the global fluent assert config in MiniLcmTests

* fix issue round tripping RichText tags where the guid would change

* merge consecutive RichSpans that have the same props

* Replicate input tab-selection handling and remove custom key handlers.

* fix issue where `newDoc.eq(editor.state.doc)` was never true because node.attr.richSpan.text was out of date

* don't use inline span nodes to avoid issues related to deleting a whole span in one block

---------

Co-authored-by: Tim Haasdyk <tim_haasdyk@sil.org>

Author: hahn-kev

backend/Directory.Packages.props

@@ -24,7 +24,6 @@
     <PackageVersion Include="HotChocolate.Data.EntityFramework" Version="14.0.0" />
     <PackageVersion Include="HotChocolate.Diagnostics" Version="14.0.0" />
     <PackageVersion Include="HotChocolate.Types.Analyzers" Version="14.0.0" />
-    <PackageVersion Include="HtmlAgilityPack" Version="1.12.0" />
     <PackageVersion Include="Humanizer.Core" Version="2.14.1" />
     <PackageVersion Include="icu.net" Version="3.0.1" />
     <PackageVersion Include="linq2db.AspNet" Version="5.4.1" />

backend/FwLite/FwDataMiniLcmBridge.Tests/FluentAssertGlobalConfig.cs

@@ -0,0 +1,13 @@
+using FluentAssertions.Extensibility;
+
+[assembly: AssertionEngineInitializer(typeof(FluentAssertGlobalConfig), nameof(FluentAssertGlobalConfig.Initialize))]
+
+namespace FwDataMiniLcmBridge.Tests;
+
+public static class FluentAssertGlobalConfig
+{
+    public static void Initialize()
+    {
+        MiniLcm.Tests.FluentAssertGlobalConfig.Initialize();
+    }
+}

backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/ExampleSentenceTests.cs

@@ -0,0 +1,12 @@
+using FwDataMiniLcmBridge.Tests.Fixtures;
+
+namespace FwDataMiniLcmBridge.Tests.MiniLcmTests;
+
+[Collection(ProjectLoaderFixture.Name)]
+public class ExampleSentenceTests(ProjectLoaderFixture fixture) : ExampleSentenceTestsBase
+{
+    protected override Task<IMiniLcmApi> NewApi()
+    {
+        return Task.FromResult<IMiniLcmApi>(fixture.NewProjectApi("ExampleSentenceTests", "en", "en"));
+    }
+}

backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs

@@ -52,18 +52,18 @@ public async Task UpdateEntry_CanUpdateExampleSentenceTranslations_WhenNoTransla
         var entry = await Api.CreateEntry(new Entry
         {
             LexemeForm = { { "en", "test" } },
-            Note = { { "en", "this is a test note" } },
+            Note = { { "en", new RichString("this is a test note") } },
             CitationForm = { { "en", "test" } },
-            LiteralMeaning = { { "en", "test" } },
+            LiteralMeaning = { { "en", new RichString("test") } },
             Senses =
             [
                 new Sense
                 {
                     Gloss = { { "en", "test" } },
-                    Definition = { { "en", "test" } },
+                    Definition = { { "en", new RichString("test") } },
                     ExampleSentences =
                     [
-                        new ExampleSentence { Sentence = { { "en", "testing is good" } } }
+                        new ExampleSentence { Sentence = { { "en", new RichString("testing is good") } } }
                     ]
                 }
             ]
@@ -86,15 +86,15 @@ public async Task UpdateEntry_CanUpdateExampleSentenceTranslations_WhenNoTransla
 
         var before = entry.Copy();
         var exampleSentence = entry.Senses[0].ExampleSentences[0];
-        exampleSentence.Translation = new() { { "en", "updated" } };
+        exampleSentence.Translation = new() { { "en", new RichString("updated") } };
 
         // Act
         var updatedEntry = await Api.UpdateEntry(before, entry);
         var updatedExampleSentence = updatedEntry.Senses[0].ExampleSentences[0];
 
         // Assert
         updatedExampleSentence.Translation.Should().ContainSingle();
-        updatedExampleSentence.Translation["en"].Should().Be("updated");
+        updatedExampleSentence.Translation["en"].Should().BeEquivalentTo(new RichString("updated", "en"));
         updatedEntry.Should().BeEquivalentTo(entry, options => options);
     }
 }

backend/FwLite/FwDataMiniLcmBridge.Tests/RichTextTests.cs

@@ -655,7 +655,11 @@ private ExampleSentence FromLexExampleSentence(Guid senseGuid, ILexExampleSenten
             Id = sentence.Guid,
             SenseId = senseGuid,
             Sentence = FromLcmMultiString(sentence.Example),
-            Reference = sentence.Reference.Text,
+            Reference =
+                sentence.Reference.Length == 0
+                    ? null
+                    : RichTextMapping.FromTsString(sentence.Reference,
+                        h => h is null ? null : (WritingSystemId?)GetWritingSystemId(h.Value)),
             Translation = translation is null ? new() : FromLcmMultiString(translation),
         };
     }
@@ -680,7 +684,11 @@ private RichMultiString FromLcmMultiString(IMultiString multiString)
         {
             var tsString = multiString.GetStringFromIndex(i, out var ws);
 
-            result.Add(GetWritingSystemId(ws), tsString.Text);
+            result.Add(GetWritingSystemId(ws), RichTextMapping.FromTsString(tsString, h =>
+            {
+                if (h is null) return null;
+                return GetWritingSystemId(h.Value);
+            }));
         }
 
         return result;
@@ -1065,7 +1073,7 @@ private void UpdateLcmMultiString(ITsMultiString multiString, RichMultiString ne
         foreach (var (ws, value) in newMultiString)
         {
             var writingSystemHandle = GetWritingSystemHandle(ws);
-            multiString.set_String(writingSystemHandle, TsStringUtils.MakeString(value, writingSystemHandle));
+            multiString.set_String(writingSystemHandle, RichTextMapping.ToTsString(value, id => GetWritingSystemHandle(id)));
         }
     }
 
@@ -1329,8 +1337,9 @@ internal void CreateExampleSentence(ILexSense lexSense, ExampleSentence exampleS
         UpdateLcmMultiString(lexExampleSentence.Example, exampleSentence.Sentence);
         var translation = CreateExampleSentenceTranslation(lexExampleSentence);
         UpdateLcmMultiString(translation.Translation, exampleSentence.Translation);
-        lexExampleSentence.Reference = TsStringUtils.MakeString(exampleSentence.Reference,
-            lexExampleSentence.Reference.get_WritingSystem(0));
+        lexExampleSentence.Reference = exampleSentence.Reference is null
+            ? null
+            : RichTextMapping.ToTsString(exampleSentence.Reference, id => GetWritingSystemHandle(id));
     }
 
     public ICmTranslation CreateExampleSentenceTranslation(ILexExampleSentence parent)

backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs

@@ -90,7 +90,10 @@ internal static int GetWritingSystemHandle(this LcmCache cache, WritingSystemId
             };
         }
 
-        var lcmWs = cache.ServiceLocator.WritingSystemManager.Get(ws.Code);
+        if (!cache.ServiceLocator.WritingSystemManager.TryGet(ws.Code, out var lcmWs))
+        {
+            throw new NullReferenceException($"unable to find writing system with id '{ws.Code}'");
+        }
         if (lcmWs is not null && type is not null)
         {
             var validWs = type switch

backend/FwLite/FwDataMiniLcmBridge/Api/LcmHelpers.cs

@@ -1,7 +1,9 @@
 using System.Collections.Frozen;
+using System.Drawing;
 using System.Globalization;
 using System.Text;
 using MiniLcm.Models;
+using MiniLcm.RichText;
 using Mono.Unix.Native;
 using SIL.LCModel.Core.KernelInterfaces;
 using SIL.LCModel.Core.Text;
@@ -125,6 +127,23 @@ public static class RichTextMapping
         };
     }
 
+    public static RichString FromTsString(ITsString tsString, Func<int?, WritingSystemId?> wsIdLookup)
+    {
+        var spans = new List<RichSpan>(tsString.RunCount);
+        for (int i = 0; i < tsString.RunCount; i++)
+        {
+            var props = tsString.FetchRunInfo(i, out _);
+            var span = new RichSpan
+            {
+                Text = tsString.get_RunText(i)
+            };
+            WriteToSpan(span, props, wsIdLookup);
+            spans.Add(span);
+        }
+
+        return new RichString([..spans]);
+    }
+
     public static void WriteToSpan(RichSpan span, ITsTextProps textProps, Func<int?, WritingSystemId?> wsIdLookup)
     {
         for (int i = 0; i < textProps.IntPropCount; i++)
@@ -176,7 +195,8 @@ private static void SetFromInt(RichSpan span, ITsTextProps textProps, FwTextProp
         switch (type)
         {
             case FwTextPropType.ktptWs:
-                span.Ws = wsIdLookup(GetNullableIntProp(textProps, type));
+                var wsHandle = GetNullableIntProp(textProps, type);
+                span.Ws = wsIdLookup(wsHandle) ?? throw new ArgumentException($"ws handle {wsHandle} is not valid");
                 break;
             case FwTextPropType.ktptBaseWs:
                 span.WsBase = wsIdLookup(GetNullableIntProp(textProps, type));
@@ -231,6 +251,20 @@ private static void SetFromInt(RichSpan span, ITsTextProps textProps, FwTextProp
         }
     }
 
+    public static ITsString ToTsString(RichString richString, Func<WritingSystemId, int> wsHandleLookup)
+    {
+        var stringBuilder = TsStringUtils.MakeIncStrBldr();
+        var propsBldr = TsStringUtils.MakePropsBldr();
+        foreach (var span in richString.Spans)
+        {
+            WriteToTextProps(span, propsBldr, wsHandleLookup);
+            stringBuilder.AppendTsString(TsStringUtils.MakeString(span.Text, propsBldr.GetTextProps()));
+            propsBldr.Clear();
+        }
+
+        return stringBuilder.GetString();
+    }
+
     public static void WriteToTextProps(RichSpan span,
         ITsPropsBldr builder,
         Func<WritingSystemId, int> wsHandleLookup)
@@ -253,8 +287,8 @@ public static void WriteToTextProps(RichSpan span,
             }
         }
 
-        if (span.Ws is not null)
-            SetInt(builder, FwTextPropType.ktptWs, wsHandleLookup(span.Ws.Value));
+
+        SetInt(builder, FwTextPropType.ktptWs, wsHandleLookup(span.Ws));
         if (span.WsBase is not null)
             SetInt(builder, FwTextPropType.ktptBaseWs, wsHandleLookup(span.WsBase.Value));
         SetInt(builder, FwTextPropType.ktptItalic, ReverseMapToggle(span.Italic));
@@ -393,12 +427,11 @@ private static int ReverseSizeUnit(RichTextSizeUnit? unit)
         {
             if (string.IsNullOrEmpty(value))
                 return null;
-            const int guidLength = 16;
-            var bytes = Encoding.Unicode.GetBytes(value).AsSpan();
-            Guid[] guids = new Guid[bytes.Length / guidLength];
+            const int guidLength = 8;
+            Guid[] guids = new Guid[value.Length / guidLength];
             for (int i = 0; i < guids.Length; i++)
             {
-                guids[i] = new Guid(bytes.Slice(i * guidLength, guidLength));
+                guids[i] = MiscUtils.GetGuidFromObjData(value.Substring(i * guidLength, guidLength));
             }
             return guids;
         }
@@ -518,26 +551,26 @@ private static string ReverseTags(Guid[] tags)
         };
     }
 
-    private static string? GetNullableColorProp(ITsTextProps textProps, FwTextPropType type)
+    private static Color? GetNullableColorProp(ITsTextProps textProps, FwTextPropType type)
     {
         if (!textProps.TryGetIntValue(type, out _, out var value))
         {
             return null;
         }
-        if (value == (int)FwTextColor.kclrTransparent) return "#00000000";
+        if (value == (int)FwTextColor.kclrTransparent) return ColorJsonConverter.UnnamedTransparent;
         int blue = (value >> 16) & 0xff;
         int green = (value >> 8) & 0xff;
         int red = value & 0xff;
-        return $"#{red:x2}{green:x2}{blue:x2}";
+        return Color.FromArgb(red, green, blue);
     }
 
-    private static int? ReverseColor(string? rgb)
+    private static int? ReverseColor(Color? rgb)
     {
-        if (string.IsNullOrEmpty(rgb))
+        if (rgb is null || rgb.Value == default)
             return null;
-        if (rgb == "#00000000")
+        if (rgb.Value.A == 0)
             return (int)FwTextColor.kclrTransparent;
-        return (int)ColorUtil.ConvertRGBtoBGR(uint.Parse(rgb.AsSpan()[1..], NumberStyles.HexNumber));
+        return (int)ColorUtil.ConvertRGBtoBGR(uint.Parse(ColorTranslator.ToHtml(rgb.Value).AsSpan()[1..], NumberStyles.HexNumber));
     }
 
     private static RichTextAlign? GetNullableRichTextAlign(ITsTextProps textProps)

backend/FwLite/FwDataMiniLcmBridge/Api/RichTextMapping.cs

@@ -19,6 +19,12 @@ public void Add(WritingSystemId key, string value)
         multiString.set_String(writingSystemHandle, TsStringUtils.MakeString(value, writingSystemHandle));
     }
 
+    public void Add(WritingSystemId key, RichString value)
+    {
+        var writingSystemHandle = lexboxLcmApi.GetWritingSystemHandle(key);
+        multiString.set_String(writingSystemHandle, RichTextMapping.ToTsString(value, ws => lexboxLcmApi.GetWritingSystemHandle(ws)));
+    }
+
     public bool ContainsKey(WritingSystemId key)
     {
         if (multiString.StringCount == 0) return false;

backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateDictionaryProxy.cs

@@ -84,12 +84,12 @@ public override MultiString Copy()
 
 public class UpdateRichMultiStringProxy(ITsMultiString multiString, FwDataMiniLcmApi lexboxLcmApi) : RichMultiString, IDictionary
 {
-    private IDictionary<WritingSystemId, string> proxy => new UpdateDictionaryProxy(multiString, lexboxLcmApi);
+    private UpdateDictionaryProxy proxy = new(multiString, lexboxLcmApi);
 
     void IDictionary.Add(object key, object? value)
     {
-        var valStr = value as string ??
-                     throw new ArgumentException($"unable to convert value {value?.GetType().Name ?? "null"} to string",
+        var valStr = value as RichString ??
+                     throw new ArgumentException($"unable to convert value {value?.GetType().Name ?? "null"} to RichString",
                          nameof(value));
         if (key is WritingSystemId keyWs)
         {