Skip to content

Commit 9c1d75e

Browse files
hahn-kevmyieye
hahn-kev
and
myieye
authored
implement accent/diacritic ignore search (#1724)
* implement accent/diacritic ignore search * use contains helper to ensure search works for FwData * update SqlHelper.SearchValue to execute matching the same as when translated into SQL * make ContainsDiacriticMatch which ignores diacritics when there's none in the search string, otherwise it does match * write a test to validate the performance of entry search. --------- Co-authored-by: Tim Haasdyk <tim_haasdyk@sil.org>
1 parent 32f4945 commit 9c1d75e

7 files changed

Lines changed: 197 additions & 8 deletions

File tree

‎backend/FwLite/FwDataMiniLcmBridge/Api/LcmHelpers.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System.Globalization;
2+
using MiniLcm.Culture;
23
using MiniLcm.Models;
34
using SIL.LCModel;
45
using SIL.LCModel.Core.KernelInterfaces;
@@ -13,7 +14,7 @@ internal static bool SearchValue(this ITsMultiString multiString, string value)
1314
{
1415
var tsString = multiString.GetStringFromIndex(i, out var _);
1516
if (string.IsNullOrEmpty(tsString.Text)) continue;
16-
if (tsString.Text.Contains(value, StringComparison.InvariantCultureIgnoreCase))
17+
if (tsString.Text.ContainsDiacriticMatch(value))
1718
{
1819
return true;
1920
}

‎backend/FwLite/LcmCrdt.Tests/MiniLcmTests/QueryEntryTests.cs‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1-
namespace LcmCrdt.Tests.MiniLcmTests;
1+
using System.Diagnostics;
2+
using Xunit.Abstractions;
23

3-
public class QueryEntryTests : QueryEntryTestsBase
4+
namespace LcmCrdt.Tests.MiniLcmTests;
5+
6+
public class QueryEntryTests(ITestOutputHelper outputHelper) : QueryEntryTestsBase
47
{
58
private readonly MiniLcmApiFixture _fixture = new();
69

@@ -11,6 +14,32 @@ protected override async Task<IMiniLcmApi> NewApi()
1114
return api;
1215
}
1316

17+
18+
19+
[Theory]
20+
[InlineData(50_000)]
21+
[InlineData(100_000)]
22+
public async Task QueryPerformanceTesting(int count)
23+
{
24+
await _fixture.Api.BulkCreateEntries(AsyncEnumerable.Range(0, count).Select(i => new Entry { LexemeForm = { ["en"] = Guid.NewGuid().ToString() } }));
25+
26+
var testIterations = 10;
27+
var startTimestamp = Stopwatch.GetTimestamp();
28+
for (int i = 0; i < testIterations; i++)
29+
{
30+
//search should not match anything as we only want to test the match performance
31+
var results = await Api.SearchEntries("asdfgbope").ToArrayAsync();
32+
results.Should().BeEmpty();
33+
}
34+
35+
var totalRuntime = Stopwatch.GetElapsedTime(startTimestamp);
36+
var queryTime = totalRuntime / testIterations;
37+
var timePerEntry = queryTime / count;
38+
outputHelper.WriteLine(
39+
$"Total query time: {queryTime.TotalMilliseconds}ms, time per entry: {timePerEntry.TotalMicroseconds}microseconds");
40+
timePerEntry.TotalMicroseconds.Should().BeLessThan(10);//on my machine I got 3.9, so this is a safe margin
41+
}
42+
1443
public override async Task DisposeAsync()
1544
{
1645
await base.DisposeAsync();

‎backend/FwLite/LcmCrdt/Data/CustomSqliteFunctionInterceptor.cs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
using System.Data.Common;
2+
using System.Globalization;
3+
using System.Runtime.CompilerServices;
4+
using System.Text;
5+
using Microsoft.Data.Sqlite;
6+
using Microsoft.EntityFrameworkCore.Diagnostics;
7+
using MiniLcm.Culture;
8+
9+
namespace LcmCrdt.Data;
10+
11+
public class CustomSqliteFunctionInterceptor : IDbConnectionInterceptor
12+
{
13+
public const string ContainsFunction = "contains";
14+
15+
public void ConnectionOpened(DbConnection connection, ConnectionEndEventData eventData)
16+
{
17+
var sqliteConnection = (SqliteConnection)connection;
18+
//creates a new function that can be used in queries
19+
sqliteConnection.CreateFunction(ContainsFunction,
20+
//in sqlite strings are byte arrays, so we can avoid allocating strings by using spans
21+
(byte[]? str, byte[]? value) =>
22+
{
23+
if (str is null || value is null) return false;
24+
25+
Span<char> source = stackalloc char[Encoding.UTF8.GetCharCount(str)];
26+
Span<char> search = stackalloc char[Encoding.UTF8.GetCharCount(value)];
27+
Encoding.UTF8.GetChars(str, source);
28+
Encoding.UTF8.GetChars(value, search);
29+
return CultureInfo.InvariantCulture.CompareInfo.IndexOf(source,
30+
search,
31+
ContainsDiacritic(search)
32+
? CompareOptions.IgnoreCase
33+
: CompareOptions.IgnoreNonSpace | CompareOptions.IgnoreCase
34+
) >= 0;
35+
});
36+
}
37+
38+
private static bool ContainsDiacritic(in ReadOnlySpan<char> value)
39+
{
40+
bool hasAccent = false;
41+
//todo we could maybe get rid of this normalization step if the text is already normalized
42+
//that would mean we could just iterate the value here rather than creating a new string
43+
foreach (var ch in new string(value).Normalize(NormalizationForm.FormD))
44+
{
45+
if (CharUnicodeInfo.GetUnicodeCategory(ch) == UnicodeCategory.NonSpacingMark)
46+
{
47+
hasAccent = true;
48+
break;
49+
}
50+
}
51+
return hasAccent;
52+
}
53+
54+
public Task ConnectionOpenedAsync(DbConnection connection,
55+
ConnectionEndEventData eventData,
56+
CancellationToken cancellationToken = new CancellationToken())
57+
{
58+
ConnectionOpened(connection, eventData);
59+
return Task.CompletedTask;
60+
}
61+
}

‎backend/FwLite/LcmCrdt/LcmCrdtDbContext.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ public class LcmCrdtDbContext(DbContextOptions<LcmCrdtDbContext> dbContextOption
1515
public IQueryable<WritingSystem> WritingSystems => Set<WritingSystem>().AsNoTracking();
1616
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
1717
{
18-
optionsBuilder.AddInterceptors(setupCollationInterceptor);
18+
optionsBuilder.AddInterceptors(setupCollationInterceptor, new CustomSqliteFunctionInterceptor());
1919
}
2020

2121
protected override void OnModelCreating(ModelBuilder modelBuilder)

‎backend/FwLite/LcmCrdt/SqlHelpers.cs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1-
using System.Linq.Expressions;
1+
using System.Globalization;
2+
using System.Linq.Expressions;
3+
using LcmCrdt.Data;
24
using LinqToDB;
5+
using LinqToDB.DataProvider.SQLite;
6+
using MiniLcm.Culture;
37

48
namespace LcmCrdt;
59

@@ -19,11 +23,14 @@ private static Expression<Func<MultiString, string, bool>> HasValueExpression()
1923
[ExpressionMethod(nameof(SearchValueExpression))]
2024
public static bool SearchValue(this MultiString ms, string search)
2125
{
22-
return ms.Values.Any(pair => pair.Value.Contains(search));
26+
return ms.Values.Any(pair => pair.Value.ContainsDiacriticMatch(search));
2327
}
2428

2529
private static Expression<Func<MultiString, string, bool>> SearchValueExpression()
2630
{
27-
return (ms, search) => Json.QueryValues(ms).Any(s => s.Contains(search));
31+
return (ms, search) => Json.QueryValues(ms).Any(s => ContainsIgnoreCaseAccents(s, search));
2832
}
33+
34+
[Sql.Expression(CustomSqliteFunctionInterceptor.ContainsFunction + "({0}, {1})")]
35+
private static bool ContainsIgnoreCaseAccents(string s, string search) => s.ContainsDiacriticMatch(search);
2936
}

‎backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
namespace MiniLcm.Tests;
1+
using System.Text;
2+
3+
namespace MiniLcm.Tests;
24

35
public abstract class QueryEntryTestsBase : MiniLcmTestBase
46
{
@@ -211,4 +213,41 @@ public async Task CanFilterExampleSentenceText()
211213
var results = await Api.GetEntries(new(Filter: new() { GridifyFilter = "Senses.ExampleSentences.Sentence[en]=*phone" })).ToArrayAsync();
212214
results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana);
213215
}
216+
217+
[Theory]
218+
[InlineData("a", "a")]
219+
[InlineData("a", "A")]
220+
[InlineData("A", "Ã")]
221+
[InlineData("ap", "apple")]
222+
[InlineData("ap", "APPLE")]
223+
[InlineData("ing", "walking")]
224+
[InlineData("ing", "WALKING")]
225+
[InlineData("Ãp", "Ãpple")]
226+
[InlineData("Ãp", "ãpple")]
227+
[InlineData("ap", "Ãpple")]
228+
public async Task SuccessfulMatches(string searchTerm, string word)
229+
{
230+
word = word.Normalize(NormalizationForm.FormD);
231+
//should we be normalizing the search term internally?
232+
searchTerm = searchTerm.Normalize(NormalizationForm.FormD);
233+
await Api.CreateEntry(new Entry { LexemeForm = { ["en"] = word } });
234+
var words = await Api.SearchEntries(searchTerm).Select(e => e.LexemeForm["en"]).ToArrayAsync();
235+
words.Should().Contain(word);
236+
}
237+
238+
[Theory]
239+
[InlineData("a", "b")]
240+
[InlineData("ab", "b")]
241+
[InlineData("Ã", "A")] // Accented should not match base
242+
[InlineData("apple", "orange")] // Completely different words
243+
[InlineData("É", "È")] // Different accents
244+
public async Task NegativeMatches(string searchTerm, string word)
245+
{
246+
word = word.Normalize(NormalizationForm.FormD);
247+
//should we be normalizing the search term internally?
248+
searchTerm = searchTerm.Normalize(NormalizationForm.FormD);
249+
await Api.CreateEntry(new Entry { LexemeForm = { ["en"] = word } });
250+
var words = await Api.SearchEntries(searchTerm).Select(e => e.LexemeForm["en"]).ToArrayAsync();
251+
words.Should().NotContain(word);
252+
}
214253
}

‎backend/FwLite/MiniLcm/Culture/StringExtensions.cs‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
using System.Globalization;
2+
using System.Text;
3+
4+
namespace MiniLcm.Culture;
5+
6+
public static class StringExtensions
7+
{
8+
public static bool Contains(this string str, string value, CultureInfo cultureInfo, CompareOptions comparison = CompareOptions.None)
9+
{
10+
return cultureInfo.CompareInfo.IndexOf(str, value, comparison) >= 0;
11+
}
12+
13+
/// <summary>
14+
/// searches a string for a match ignoring diacritics, but only when the search string does not contain diacritics
15+
/// </summary>
16+
/// <param name="str">source of the search</param>
17+
/// <param name="search">string to search for</param>
18+
public static bool ContainsDiacriticMatch(this string str, string search)
19+
{
20+
if (ContainsDiacritic(search))
21+
{
22+
return Contains(str, search, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase);
23+
}
24+
25+
return Contains(str,
26+
search,
27+
CultureInfo.InvariantCulture,
28+
CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace);
29+
}
30+
31+
public static bool ContainsDiacritic(string value)
32+
{
33+
bool hasAccent = false;
34+
foreach (var ch in value.Normalize(NormalizationForm.FormD))
35+
{
36+
if (CharUnicodeInfo.GetUnicodeCategory(ch) == UnicodeCategory.NonSpacingMark)
37+
{
38+
hasAccent = true;
39+
break;
40+
}
41+
}
42+
return hasAccent;
43+
}
44+
45+
public static bool Equals(this string str,
46+
string value,
47+
CultureInfo cultureInfo,
48+
CompareOptions comparison = CompareOptions.None)
49+
{
50+
return cultureInfo.CompareInfo.Compare(str, value, comparison) == 0;
51+
}
52+
}

0 commit comments

Comments
 (0)