Added compatibility normalization to UString. Updated to version 0.9.1.

Author: FoolRunning

UnicodeHelper.Tests/TestData/NormalizationTestDataSet.cs

@@ -8,17 +8,16 @@
 
 namespace UnicodeHelper.TestData
 {
-    public sealed record NormalizationTestData(UString Source, UString NfcResult, UString NfdResult, string Description)
+    public sealed record NormalizationTestData(UString Source, 
+        UString NfcResult, UString NfdResult, UString NfkcResult, UString NfkdResult, 
+        string Description)
     {
         public override string ToString()
         {
             return Description;
         }
     }
 
-    /// <summary>
-    /// 
-    /// </summary>
     /// <remarks>Test data taken from https://www.unicode.org/Public/UCD/latest/ucd/NormalizationTest.txt</remarks>
     internal static class NormalizationTestDataSet
     {
@@ -55,6 +54,8 @@ static NormalizationTestDataSet()
                 testCases.Add(new NormalizationTestData(CreateUStringFromCodepoints(line.Source), 
                     CreateUStringFromCodepoints(line.NfcResult),
                     CreateUStringFromCodepoints(line.NfdResult), 
+                    CreateUStringFromCodepoints(line.NfkcResult),
+                    CreateUStringFromCodepoints(line.NfkdResult), 
                     string.Join("", line.Comments).TrimStart(' ', '#')));
             }
         }

UnicodeHelper.Tests/UStringTests.cs

@@ -738,7 +738,8 @@ public void Normalization_FormC()
             foreach (NormalizationTestData testData in NormalizationTestDataSet.TestCases)
             {
                 UString result = testData.Source.Normalize(NormalizationForm.FormC);
-                Assert.AreEqual(testData.NfcResult, result, testData.Description);
+                Assert.AreEqual(testData.NfcResult, result, 
+                    ErrorString(testData.NfcResult, result, testData.Description));
             }
         }
 
@@ -748,7 +749,30 @@ public void Normalization_FormD()
             foreach (NormalizationTestData testData in NormalizationTestDataSet.TestCases)
             {
                 UString result = testData.Source.Normalize(NormalizationForm.FormD);
-                Assert.AreEqual(testData.NfdResult, result, testData.Description);
+                Assert.AreEqual(testData.NfdResult, result, 
+                    ErrorString(testData.NfdResult, result, testData.Description));
+            }
+        }
+        
+        [TestMethod]
+        public void Normalization_FormKD()
+        {
+            foreach (NormalizationTestData testData in NormalizationTestDataSet.TestCases)
+            {
+                UString result = testData.Source.Normalize(NormalizationForm.FormKD);
+                Assert.AreEqual(testData.NfkdResult, result, 
+                    ErrorString(testData.NfkdResult, result, testData.Description));
+            }
+        }
+
+        [TestMethod]
+        public void Normalization_FormKC()
+        {
+            foreach (NormalizationTestData testData in NormalizationTestDataSet.TestCases)
+            {
+                UString result = testData.Source.Normalize(NormalizationForm.FormKC);
+                Assert.AreEqual(testData.NfkcResult, result,
+                    ErrorString(testData.NfkcResult, result, testData.Description));
             }
         }
         #endregion
@@ -764,6 +788,16 @@ private static UString CreateTestSubstring(string? str)
             UString us = new(pre + str + suff);
             return us.SubString(pre.Length, us.Length - pre.Length - suff.Length);
         }
+
+        private static string ErrorString(UString expected, UString result, string description)
+        {
+            return $"\nExp: {UStrToCodepoints(expected)}\nGot: {UStrToCodepoints(result)}\nFor:{description}";
+        }
+
+        private static string UStrToCodepoints(UString ustr)
+        {
+            return string.Join(" ", ustr.Select(uc => uc.ToHexString()));
+        }
         #endregion
     }
 }

UnicodeHelper/Internal/CompositionExclusions.cs

@@ -6,11 +6,10 @@
 
 namespace UnicodeHelper.Internal
 {
-    /// <summary>
-    /// 
-    /// </summary>
     /// <remarks>This class represents the data in the Unicode specification
     /// <see href="https://www.unicode.org/reports/tr44/#CompositionExclusions.txt">CompositionExclusions.txt</see></remarks>
+    /// <remarks>This class is designed to be used and thrown away since the data is only needed during the creation
+    /// of the composition mappings.</remarks>
     internal sealed class CompositionExclusions
     {
         #region Data fields

UnicodeHelper/Internal/HelperUtils.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 
 namespace UnicodeHelper.Internal
 {
@@ -39,5 +40,37 @@ public static TextDirection DetermineDirection(IEnumerable<UCodepoint> codepoint
 
             return TextDirection.Undefined;
         }
+
+        /// <summary>
+        /// Efficiently converts a bool to an int (0 or 1).
+        /// </summary>
+        /// <remarks>Taken from https://stackoverflow.com/a/66993553 </remarks>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static unsafe int BoolToInt(bool b)
+        {
+            return *(byte*)&b;
+        }
+
+        /// <summary>
+        /// Sorts the specified decomposed character array in Unicode canonical order (based on combining class).
+        /// </summary>
+        public static void SortCanonical(UCodepoint[] decomposedChar, int count)
+        {
+            for (int i = 1; i < count; i++)
+            {
+                byte ucClass = UnicodeData.GetCombiningClass(decomposedChar[i]);
+                if (ucClass == 0)
+                    continue;
+
+                byte ucClassPrev = UnicodeData.GetCombiningClass(decomposedChar[i - 1]);
+                if (ucClassPrev <= ucClass) 
+                    continue;
+
+                // Swap items
+                (decomposedChar[i], decomposedChar[i - 1]) = (decomposedChar[i - 1], decomposedChar[i]);
+                if (i > 1)
+                    i -= 2; // Re-evaluate previous items
+            }
+        }
     }
 }