implement accent/diacritic ignore search

backend/FwLite/FwDataMiniLcmBridge/Api/LcmHelpers.cs

@@ -1,4 +1,5 @@
 using System.Globalization;
+using MiniLcm.Culture;
 using MiniLcm.Models;
 using SIL.LCModel;
 using SIL.LCModel.Core.KernelInterfaces;
@@ -13,7 +14,7 @@ internal static bool SearchValue(this ITsMultiString multiString, string value)
         {
             var tsString = multiString.GetStringFromIndex(i, out var _);
             if (string.IsNullOrEmpty(tsString.Text)) continue;
-            if (tsString.Text.Contains(value, StringComparison.InvariantCultureIgnoreCase))
+            if (tsString.Text.Contains(value, CultureInfo.InvariantCulture, CompareOptions.IgnoreNonSpace | CompareOptions.IgnoreCase))
             {
                 return true;
             }

backend/FwLite/LcmCrdt/Data/CustomSqliteFunctionInterceptor.cs

@@ -0,0 +1,39 @@
+using System.Data.Common;
+using System.Globalization;
+using System.Text;
+using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore.Diagnostics;
+using MiniLcm.Culture;
+
+namespace LcmCrdt.Data;
+
+public class CustomSqliteFunctionInterceptor : IDbConnectionInterceptor
+{
+    public const string ContainsFunction = "contains";
+    public void ConnectionOpened(DbConnection connection, ConnectionEndEventData eventData)
+    {
+        var sqliteConnection = (SqliteConnection)connection;
+        //creates a new function that can be used in queries
+        sqliteConnection.CreateFunction(ContainsFunction,
+            //in sqlite strings are byte arrays, so we can avoid allocating strings by using spans
+            (byte[]? str, byte[]? value) =>
+            {
+                if (str is null || value is null) return false;
+                Span<char> source = stackalloc char[Encoding.UTF8.GetCharCount(str)];
+                Span<char> search = stackalloc char[Encoding.UTF8.GetCharCount(value)];
+                Encoding.UTF8.GetChars(str, source);
+                Encoding.UTF8.GetChars(value, search);
+                return CultureInfo.InvariantCulture.CompareInfo.IndexOf(source,
+                    search,
+                    CompareOptions.IgnoreNonSpace | CompareOptions.IgnoreCase) >= 0;
+            });
+    }
+
+    public Task ConnectionOpenedAsync(DbConnection connection,
+        ConnectionEndEventData eventData,
+        CancellationToken cancellationToken = new CancellationToken())
+    {
+        ConnectionOpened(connection, eventData);
+        return Task.CompletedTask;
+    }
+}

backend/FwLite/LcmCrdt/LcmCrdtDbContext.cs

@@ -15,7 +15,7 @@ public class LcmCrdtDbContext(DbContextOptions<LcmCrdtDbContext> dbContextOption
     public IQueryable<WritingSystem> WritingSystems => Set<WritingSystem>().AsNoTracking();
     protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
     {
-        optionsBuilder.AddInterceptors(setupCollationInterceptor);
+        optionsBuilder.AddInterceptors(setupCollationInterceptor, new CustomSqliteFunctionInterceptor());
     }
 
     protected override void OnModelCreating(ModelBuilder modelBuilder)

backend/FwLite/LcmCrdt/SqlHelpers.cs

@@ -1,5 +1,9 @@
-using System.Linq.Expressions;
+using System.Globalization;
+using System.Linq.Expressions;
+using LcmCrdt.Data;
 using LinqToDB;
+using LinqToDB.DataProvider.SQLite;
+using MiniLcm.Culture;
 
 namespace LcmCrdt;
 
@@ -19,11 +23,15 @@ private static Expression<Func<MultiString, string, bool>> HasValueExpression()
     [ExpressionMethod(nameof(SearchValueExpression))]
     public static bool SearchValue(this MultiString ms, string search)
     {
-        return ms.Values.Any(pair => pair.Value.Contains(search));
+        return ms.Values.Any(pair => pair.Value.Contains(search, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace));
     }
 
     private static Expression<Func<MultiString, string, bool>> SearchValueExpression()
     {
-        return (ms, search) => Json.QueryValues(ms).Any(s => s.Contains(search));
+        return (ms, search) => Json.QueryValues(ms).Any(s => ContainsIgnoreCaseAccents(s, search));
     }
+
+    [Sql.Expression(CustomSqliteFunctionInterceptor.ContainsFunction + "({0}, {1})")]
+    private static bool ContainsIgnoreCaseAccents(string s, string search) =>
+        s.Contains(search, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace);
 }

backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs

@@ -1,4 +1,6 @@
-namespace MiniLcm.Tests;
+using System.Text;
+
+namespace MiniLcm.Tests;
 
 public abstract class QueryEntryTestsBase : MiniLcmTestBase
 {
@@ -211,4 +213,42 @@ public async Task CanFilterExampleSentenceText()
         var results = await Api.GetEntries(new(Filter: new() { GridifyFilter = "Senses.ExampleSentences.Sentence[en]=*phone" })).ToArrayAsync();
         results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana);
     }
+
+    [Theory]
+    [InlineData("a", "a")]
+    [InlineData("a", "A")]
+    [InlineData("Ã", "A")] // todo: make it fail
+    [InlineData("A", "Ã")]
+    [InlineData("ap", "apple")]
+    [InlineData("ap", "APPLE")]
+    [InlineData("ing", "walking")]
+    [InlineData("ing", "WALKING")]
+    [InlineData("Ãp", "Ãpple")]
+    [InlineData("ap", "Ãpple")]
+    public async Task SuccessfulMatches(string searchTerm, string word)
+    {
+        word = word.Normalize(NormalizationForm.FormD);
+        //should we be normalizing the search term internally?
+        searchTerm = searchTerm.Normalize(NormalizationForm.FormD);
+        await Api.CreateEntry(new Entry { LexemeForm = { ["en"] = word } });
+        var words = await Api.SearchEntries(searchTerm).Select(e => e.LexemeForm["en"]).ToArrayAsync();
+        words.Should().Contain(word);
+    }
+
+    [Theory]
+    [InlineData("a", "b")]
+    [InlineData("ab", "b")]
+    [InlineData("apple", "orange")]
+    // todo: make them pass
+    // [InlineData("Ã", "A")]
+    // [InlineData("É", "È")]
+    public async Task NegativeMatches(string searchTerm, string word)
+    {
+        word = word.Normalize(NormalizationForm.FormD);
+        //should we be normalizing the search term internally?
+        searchTerm = searchTerm.Normalize(NormalizationForm.FormD);
+        await Api.CreateEntry(new Entry { LexemeForm = { ["en"] = word } });
+        var words = await Api.SearchEntries(searchTerm).Select(e => e.LexemeForm["en"]).ToArrayAsync();
+        words.Should().NotContain(word);
+    }
 }

backend/FwLite/MiniLcm/Culture/StringExtensions.cs

@@ -0,0 +1,19 @@
+using System.Globalization;
+
+namespace MiniLcm.Culture;
+
+public static class StringExtensions
+{
+    public static bool Contains(this string str, string value, CultureInfo cultureInfo, CompareOptions comparison = CompareOptions.None)
+    {
+        return cultureInfo.CompareInfo.IndexOf(str, value, comparison) >= 0;
+    }
+
+    public static bool Equals(this string str,
+        string value,
+        CultureInfo cultureInfo,
+        CompareOptions comparison = CompareOptions.None)
+    {
+        return cultureInfo.CompareInfo.Compare(str, value, comparison) == 0;
+    }
+}