Skip to content

Commit c65ba79

Browse files
FoolRunningFoolRunning
committed
Added Unicode normalization to UString. Updated to 0.9.0.
1 parent 66fa1c7 commit c65ba79

15 files changed

Lines changed: 20859 additions & 29 deletions

‎UnicodeHelper.Tests/TestData/NormalizationTest.txt‎

Lines changed: 20026 additions & 0 deletions
Large diffs are not rendered by default.

‎UnicodeHelper.Tests/TestData/NormalizationTestDataSet.cs‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
using CsvHelper;
2+
using System.Diagnostics;
3+
using System.Globalization;
4+
using System.Reflection;
5+
using CsvHelper.Configuration;
6+
using CsvHelper.Configuration.Attributes;
7+
using JetBrains.Annotations;
8+
9+
namespace UnicodeHelper.TestData
10+
{
11+
public sealed record NormalizationTestData(UString Source, UString NfcResult, UString NfdResult, string Description)
12+
{
13+
public override string ToString()
14+
{
15+
return Description;
16+
}
17+
}
18+
19+
/// <summary>
20+
///
21+
/// </summary>
22+
/// <remarks>Test data taken from https://www.unicode.org/Public/UCD/latest/ucd/NormalizationTest.txt</remarks>
23+
internal static class NormalizationTestDataSet
24+
{
25+
#region Data fields
26+
private static readonly UStringBuilder dataBldr = new();
27+
private static readonly List<NormalizationTestData> testCases = new();
28+
#endregion
29+
30+
#region Static constructor
31+
static NormalizationTestDataSet()
32+
{
33+
Stream? dataStream = Assembly.GetExecutingAssembly()
34+
.GetManifestResourceStream("UnicodeHelper.TestData.NormalizationTest.txt");
35+
Debug.Assert(dataStream != null, "Unable to find embedded test data");
36+
37+
CsvConfiguration config = new CsvConfiguration(CultureInfo.InvariantCulture)
38+
{
39+
HasHeaderRecord = false,
40+
Delimiter = ";",
41+
AllowComments = false,
42+
IgnoreBlankLines = true,
43+
Mode = CsvMode.NoEscape,
44+
TrimOptions = TrimOptions.None,
45+
MissingFieldFound = null
46+
};
47+
48+
using TextReader textReader = new StreamReader(dataStream);
49+
using CsvReader reader = new CsvReader(textReader, config);
50+
foreach (TestDataLine line in reader.GetRecords<TestDataLine>())
51+
{
52+
if (line.Source.StartsWith('#') || line.Source.StartsWith('@'))
53+
continue;
54+
55+
testCases.Add(new NormalizationTestData(CreateUStringFromCodepoints(line.Source),
56+
CreateUStringFromCodepoints(line.NfcResult),
57+
CreateUStringFromCodepoints(line.NfdResult),
58+
string.Join("", line.Comments).TrimStart(' ', '#')));
59+
}
60+
}
61+
#endregion
62+
63+
#region Properties
64+
public static IEnumerable<NormalizationTestData> TestCases => testCases;
65+
#endregion
66+
67+
#region Helper methods
68+
private static UString CreateUStringFromCodepoints(string? codepoints)
69+
{
70+
if (codepoints == null)
71+
return UString.Empty;
72+
73+
dataBldr.Clear();
74+
75+
foreach (string part in codepoints.Split(' ', StringSplitOptions.RemoveEmptyEntries))
76+
{
77+
int cp = int.Parse(part, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
78+
dataBldr.Append((UCodepoint)cp);
79+
}
80+
81+
return dataBldr.ToUString();
82+
}
83+
#endregion
84+
85+
#region TestDataLine class
86+
[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
87+
private sealed class TestDataLine
88+
{
89+
[Index(0)]
90+
public required string Source { get; set; }
91+
92+
[Index(1)]
93+
public string? NfcResult { get; set; }
94+
95+
[Index(2)]
96+
public string? NfdResult { get; set; }
97+
98+
[Index(3)]
99+
public string? NfkcResult { get; set; }
100+
101+
[Index(4)]
102+
public string? NfkdResult { get; set; }
103+
104+
[Index(5, 20)]
105+
public required string?[] Comments { get; set; }
106+
}
107+
#endregion
108+
}
109+
}

‎UnicodeHelper.Tests/UStringBuilderTests.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public class UStringBuilderTests
1616
[DynamicData(nameof(AppendUCodepointTestData))]
1717
public void Append_UCodepoint(string expectedResult, params string[] parts)
1818
{
19-
UStringBuilder usb = new UStringBuilder();
19+
using UStringBuilder usb = new UStringBuilder();
2020
foreach (string part in parts)
2121
usb.Append(UCodepoint.ReadFromStr(part, 0));
2222

@@ -43,7 +43,7 @@ public void Append_UCodepoint(string expectedResult, params string[] parts)
4343
[DynamicData(nameof(AppendUStringTestData))]
4444
public void Append_UString(string expectedResult, params string?[] parts)
4545
{
46-
UStringBuilder usb = new UStringBuilder();
46+
using UStringBuilder usb = new UStringBuilder();
4747
foreach (string? part in parts)
4848
usb.Append(part != null ? new UString(part) : null);
4949

@@ -70,7 +70,7 @@ public void Append_UString(string expectedResult, params string?[] parts)
7070
[DynamicData(nameof(AppendStringTestData))]
7171
public void Append_String(string expectedResult, params string?[] parts)
7272
{
73-
UStringBuilder usb = new UStringBuilder();
73+
using UStringBuilder usb = new UStringBuilder();
7474
foreach (string? part in parts)
7575
usb.Append(part);
7676

‎UnicodeHelper.Tests/UStringTests.cs‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
namespace UnicodeHelper
1+
using System.Text;
2+
using UnicodeHelper.TestData;
3+
4+
namespace UnicodeHelper
25
{
36
[TestClass]
47
public class UStringTests
@@ -728,6 +731,28 @@ public void DetermineDirection(string? testStr, TextDirection expectedDirection)
728731
}
729732
#endregion
730733

734+
#region Normalization tests
735+
[TestMethod]
736+
public void Normalization_FormC()
737+
{
738+
foreach (NormalizationTestData testData in NormalizationTestDataSet.TestCases)
739+
{
740+
UString result = testData.Source.Normalize(NormalizationForm.FormC);
741+
Assert.AreEqual(testData.NfcResult, result, testData.Description);
742+
}
743+
}
744+
745+
[TestMethod]
746+
public void Normalization_FormD()
747+
{
748+
foreach (NormalizationTestData testData in NormalizationTestDataSet.TestCases)
749+
{
750+
UString result = testData.Source.Normalize(NormalizationForm.FormD);
751+
Assert.AreEqual(testData.NfdResult, result, testData.Description);
752+
}
753+
}
754+
#endregion
755+
731756
#region Private helper methods
732757
private static UString CreateTestSubstring(string? str)
733758
{

‎UnicodeHelper.Tests/UnicodeHelper.Tests.csproj‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@
1111
</PropertyGroup>
1212

1313
<ItemGroup>
14+
<None Remove="TestData\NormalizationTest.txt" />
15+
</ItemGroup>
16+
17+
<ItemGroup>
18+
<EmbeddedResource Include="TestData\NormalizationTest.txt" />
19+
</ItemGroup>
20+
21+
<ItemGroup>
22+
<PackageReference Include="JetBrains.Annotations" Version="2024.3.0" />
1423
<PackageReference Include="Microsoft.Bcl.HashCode" Version="6.0.0" />
1524
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
1625
<PackageReference Include="MSTest.TestAdapter" Version="3.11.0" />

‎UnicodeHelper/DotNetStringExtensions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ public bool MoveNext()
9393
public void Reset()
9494
{
9595
_index = -1;
96-
_current = UCodepoint.MinValue;
96+
_current = UCodepoint.Null;
9797
}
9898

9999
public IEnumerator<UCodepoint> GetEnumerator()

‎UnicodeHelper/Internal/CompositionExclusions.cs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
using System.Collections.Generic;
2+
using System.IO;
3+
using CsvHelper;
4+
using CsvHelper.Configuration.Attributes;
5+
using JetBrains.Annotations;
6+
7+
namespace UnicodeHelper.Internal
8+
{
9+
/// <summary>
10+
///
11+
/// </summary>
12+
/// <remarks>This class represents the data in the Unicode specification
13+
/// <see href="https://www.unicode.org/reports/tr44/#CompositionExclusions.txt">CompositionExclusions.txt</see></remarks>
14+
internal sealed class CompositionExclusions
15+
{
16+
#region Data fields
17+
private readonly HashSet<UCodepoint> _exclusions = new HashSet<UCodepoint>();
18+
#endregion
19+
20+
#region Constructor
21+
public CompositionExclusions()
22+
{
23+
DataHelper.ReadResource("CompositionExclusions.txt", Load);
24+
}
25+
#endregion
26+
27+
#region Initialization
28+
private void Load(TextReader textReader)
29+
{
30+
using (CsvReader reader = new CsvReader(textReader, DataHelper.CsvConfiguration))
31+
{
32+
foreach (ExclusionFileLine line in reader.GetRecords<ExclusionFileLine>())
33+
{
34+
string codepoint = line.Codepoint;
35+
int commentIndex = codepoint.IndexOf('#');
36+
if (commentIndex >= 0)
37+
codepoint = codepoint.Substring(0, commentIndex).Trim();
38+
39+
_exclusions.Add(UCodepoint.FromHexStr(codepoint));
40+
}
41+
}
42+
}
43+
#endregion
44+
45+
#region Public methods
46+
public bool IsExcluded(UCodepoint codepoint)
47+
{
48+
return _exclusions.Contains(codepoint);
49+
}
50+
#endregion
51+
52+
#region ExclusionFileLine class
53+
[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
54+
private sealed class ExclusionFileLine
55+
{
56+
[Index(0)]
57+
public string Codepoint { get; set; }
58+
}
59+
#endregion
60+
}
61+
}

0 commit comments

Comments
 (0)