Sort prefix matches above other contain matches (#2163)

* Sort prefix matches above other contain matches

* Optimize fwdata sorting

Author: myieye

backend/FwLite/FwDataMiniLcmBridge/Api/Sorting.cs

@@ -7,25 +7,32 @@ namespace FwDataMiniLcmBridge.Api;
 internal static class Sorting
 {
     /// <summary>
-    /// crude emulation of FTS search relevance
+    /// Rough emulation of FTS search relevance. Headword matches come first, preferring
+    /// prefix matches (e.g. when searching "tan" then "tanan" is before "matan"), then shorter, then alphabetical.
+    /// See also: EntrySearchService.FilterAndRank for the FTS-based equivalent in LcmCrdt.
     /// </summary>
     public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, string? query = null)
     {
+        var projected = entries.Select(e => (Entry: e, Headword: e.LexEntryHeadword(sortWsHandle)));
         if (order.Ascending)
         {
-            return entries
-                .OrderByDescending(e => !string.IsNullOrEmpty(query) && (e.LexEntryHeadword(sortWsHandle)?.ContainsDiacriticMatch(query!) ?? false))
-                .ThenBy(e => e.LexEntryHeadword(sortWsHandle)?.Length ?? 0)
-                .ThenBy(e => e.LexEntryHeadword(sortWsHandle))
-                .ThenBy(e => e.Id.Guid);
+            return projected
+                .OrderByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
+                .ThenByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .ThenBy(x => x.Headword?.Length ?? 0)
+                .ThenBy(x => x.Headword)
+                .ThenBy(x => x.Entry.Id.Guid)
+                .Select(x => x.Entry);
         }
         else
         {
-            return entries
-                .OrderBy(e => !string.IsNullOrEmpty(query) && (e.LexEntryHeadword(sortWsHandle)?.ContainsDiacriticMatch(query!) ?? false))
-                .ThenByDescending(e => e.LexEntryHeadword(sortWsHandle)?.Length ?? 0)
-                .ThenByDescending(e => e.LexEntryHeadword(sortWsHandle))
-                .ThenByDescending(e => e.Id.Guid);
+            return projected
+                .OrderBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
+                .ThenBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .ThenByDescending(x => x.Headword?.Length ?? 0)
+                .ThenByDescending(x => x.Headword)
+                .ThenByDescending(x => x.Entry.Id.Guid)
+                .Select(x => x.Entry);
         }
     }
 }

backend/FwLite/LcmCrdt/Data/CustomSqliteFunctionInterceptor.cs

@@ -10,6 +10,7 @@ namespace LcmCrdt.Data;
 public class CustomSqliteFunctionInterceptor : IDbConnectionInterceptor, IConnectionInterceptor
 {
     public const string ContainsFunction = "contains";
+    public const string StartsWithFunction = "startsWith";
 
     public void ConnectionOpened(DbConnection connection, ConnectionEndEventData eventData)
     {
@@ -49,15 +50,13 @@ private void ConnectionOpened(DbConnection connection)
     {
         if (connection is SqliteConnection sqliteConnection)
         {
-            RegisterContainsFunction(sqliteConnection);
+            RegisterCustomFunctions(sqliteConnection);
         }
     }
 
-    public static void RegisterContainsFunction(SqliteConnection sqliteConnection)
+    public static void RegisterCustomFunctions(SqliteConnection sqliteConnection)
     {
-        //creates a new function that can be used in queries
         sqliteConnection.CreateFunction(ContainsFunction,
-            //in sqlite strings are byte arrays, so we can avoid allocating strings by using spans
             (byte[]? str, byte[]? value) =>
             {
                 if (str is null || value is null) return false;
@@ -66,13 +65,29 @@ public static void RegisterContainsFunction(SqliteConnection sqliteConnection)
                 Span<char> search = stackalloc char[Encoding.UTF8.GetCharCount(value)];
                 Encoding.UTF8.GetChars(str, source);
                 Encoding.UTF8.GetChars(value, search);
-                return CultureInfo.InvariantCulture.CompareInfo.IndexOf(source,
-                    search,
-                    ContainsDiacritic(search)
-                        ? CompareOptions.IgnoreCase
-                        : CompareOptions.IgnoreNonSpace | CompareOptions.IgnoreCase
-                ) >= 0;
+                var options = DiacriticMatchOptions(search);
+                return CultureInfo.InvariantCulture.CompareInfo.IndexOf(source, search, options) >= 0;
             });
+
+        sqliteConnection.CreateFunction(StartsWithFunction,
+            (byte[]? str, byte[]? value) =>
+            {
+                if (str is null || value is null) return false;
+
+                Span<char> source = stackalloc char[Encoding.UTF8.GetCharCount(str)];
+                Span<char> search = stackalloc char[Encoding.UTF8.GetCharCount(value)];
+                Encoding.UTF8.GetChars(str, source);
+                Encoding.UTF8.GetChars(value, search);
+                var options = DiacriticMatchOptions(search);
+                return CultureInfo.InvariantCulture.CompareInfo.IsPrefix(source, search, options);
+            });
+    }
+
+    private static CompareOptions DiacriticMatchOptions(in ReadOnlySpan<char> search)
+    {
+        return ContainsDiacritic(search)
+            ? CompareOptions.IgnoreCase
+            : CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace;
     }
 
     private static bool ContainsDiacritic(in ReadOnlySpan<char> value)

backend/FwLite/LcmCrdt/Data/Sorting.cs

@@ -3,14 +3,17 @@ namespace LcmCrdt.Data;
 public static class Sorting
 {
     /// <summary>
-    /// crude emulation of FTS search relevance
+    /// Rough search relevance for when FTS is unavailable. Headword matches come first, preferring
+    /// prefix matches (e.g. when searching "tan" then "tanan" is before "matan"), then shorter, then alphabetical.
+    /// See also: <see cref="FullTextSearch.EntrySearchService.FilterAndRank"/> for the FTS-based equivalent.
     /// </summary>
     public static IQueryable<Entry> ApplyRoughBestMatchOrder(this IQueryable<Entry> entries, SortOptions order, string? query = null)
     {
         if (order.Ascending)
         {
             return entries
                 .OrderByDescending(e => !string.IsNullOrEmpty(query) && SqlHelpers.ContainsIgnoreCaseAccents(e.Headword(order.WritingSystem), query!))
+                .ThenByDescending(e => !string.IsNullOrEmpty(query) && SqlHelpers.StartsWithIgnoreCaseAccents(e.Headword(order.WritingSystem), query!))
                 .ThenBy(e => e.Headword(order.WritingSystem).Length)
                 .ThenBy(e => e.Headword(order.WritingSystem))
                 .ThenBy(e => e.Id);
@@ -19,6 +22,7 @@ public static IQueryable<Entry> ApplyRoughBestMatchOrder(this IQueryable<Entry>
         {
             return entries
                 .OrderBy(e => !string.IsNullOrEmpty(query) && SqlHelpers.ContainsIgnoreCaseAccents(e.Headword(order.WritingSystem), query!))
+                .ThenBy(e => !string.IsNullOrEmpty(query) && SqlHelpers.StartsWithIgnoreCaseAccents(e.Headword(order.WritingSystem), query!))
                 .ThenByDescending(e => e.Headword(order.WritingSystem).Length)
                 .ThenByDescending(e => e.Headword(order.WritingSystem))
                 .ThenByDescending(e => e.Id);

backend/FwLite/LcmCrdt/FullTextSearch/EntrySearchService.cs

@@ -34,32 +34,31 @@ public IQueryable<Entry> Filter(IQueryable<Entry> queryable, string query)
         return FilterInternal(queryable, query).Select(t => t.Entry);
     }
 
+    /// <summary>
+    /// Filters and ranks entries using FTS. Headword matches come first, preferring prefix matches
+    /// (e.g. when searching "tan" then "tanan" is before "matan"), then shorter headwords, then alphabetical.
+    /// Non-headword matches (gloss, definition) fall back to FTS rank.
+    /// See also: <see cref="Sorting.ApplyRoughBestMatchOrder"/> for the non-FTS equivalent.
+    /// </summary>
     public IQueryable<Entry> FilterAndRank(IQueryable<Entry> queryable,
         string query,
         WritingSystemId wsId)
     {
         var filtered = FilterInternal(queryable, query);
         var ordered = filtered
-            .OrderBy(t =>
-                // First headword matches... (this allows headword matches to trump long text penalizations)
-                t.HeadwordMatches
-                // ...in order of length (e.g. so exact matches are first).
-                ? t.SearchRecord.Headword.Length
-                // Everything else falls back to FTS rank.
-                : int.MaxValue)
+            .OrderByDescending(t => t.HeadwordMatches)
+            .ThenByDescending(t => t.HeadwordPrefixMatches)
+            .ThenBy(t => t.HeadwordMatches ? t.SearchRecord.Headword.Length : int.MaxValue)
             .ThenBy(t =>
-                // For headword matches of identical length...
                 t.HeadwordMatches
-                // ...order by headword text (this prevents confusing results like "maap", "baap", "maap").
                 ? t.SearchRecord.Headword.CollateUnicode(wsId)
-                // Everything else falls back to FTS rank.
                 : string.Empty)
             .ThenBy(t => Sql.Ext.SQLite().Rank(t.SearchRecord)).ThenBy(t => t.Entry.Id);
 
         return ordered.Select(t => t.Entry);
     }
 
-    private sealed record FilterProjection(Entry Entry, EntrySearchRecord SearchRecord, bool HeadwordMatches);
+    private sealed record FilterProjection(Entry Entry, EntrySearchRecord SearchRecord, bool HeadwordMatches, bool HeadwordPrefixMatches);
 
     private IQueryable<FilterProjection> FilterInternal(IQueryable<Entry> queryable, string query)
     {
@@ -74,7 +73,8 @@ where Sql.Ext.SQLite().Match(searchRecord, ftsString) &&
                 || entry.CitationForm.SearchValue(query)
                 || entry.Senses.Any(s => s.Gloss.SearchValue(query)))
             let headwordMatches = SqlHelpers.ContainsIgnoreCaseAccents(searchRecord.Headword, query)
-            select new FilterProjection(entry, searchRecord, headwordMatches);
+            let headwordPrefixMatches = SqlHelpers.StartsWithIgnoreCaseAccents(searchRecord.Headword, query)
+            select new FilterProjection(entry, searchRecord, headwordMatches, headwordPrefixMatches);
     }
 
     private static string ToFts5LiteralString(string query)

backend/FwLite/LcmCrdt/SqlHelpers.cs

@@ -33,4 +33,7 @@ private static Expression<Func<MultiString, string, bool>> SearchValueExpression
 
     [Sql.Expression(CustomSqliteFunctionInterceptor.ContainsFunction + "({0}, {1})")]
     public static bool ContainsIgnoreCaseAccents(string s, string search) => s.ContainsDiacriticMatch(search);
+
+    [Sql.Expression(CustomSqliteFunctionInterceptor.StartsWithFunction + "({0}, {1})")]
+    public static bool StartsWithIgnoreCaseAccents(string s, string search) => s.StartsWithDiacriticMatch(search);
 }

backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs

@@ -439,10 +439,10 @@ public async Task NegativeMatches(string searchTerm, string word)
     [InlineData("word1", "word1", "word1")]
     [InlineData("app", "app,apple,banana", "app,apple")]
     [InlineData("apple", "app,apple,banana", "apple")]
-    [InlineData("att", "battery,att,attack,zatt,rap:pratt", "att,zatt,attack,battery,rap")]
+    [InlineData("att", "battery,att,attack,zatt,rap:pratt", "att,attack,zatt,battery,rap")]
     [InlineData("a", "a,da,ma,aa,c:a,ti:a", "a,aa,da,ma,c,ti")]//test non fts search
     [InlineData("ap", "app,apple,banana", "app,apple")]//test non fts search
-    [InlineData("at", "battery,att,attack,zatt,rap:pratt", "att,zatt,attack,battery,rap")] //test non fts search
+    [InlineData("at", "battery,att,attack,zatt,rap:pratt", "att,attack,zatt,battery,rap")] //test non fts search
     // matching headwords trump glosses, even if the headword match is penalized for other long fields
     [InlineData("aap", "maap-aap,maap,liaap,aap:to-penalize-the-bm25-rank-this-gloss-is-very-very-very-very-very-very-very-very-very-long", "aap,maap,liaap,maap-aap")]
     // matching headwords of the same length are ordered alphabetically

backend/FwLite/MiniLcm/Culture/StringExtensions.cs

@@ -5,27 +5,36 @@ namespace MiniLcm.Culture;
 
 public static class StringExtensions
 {
-    public static bool Contains(this string str, string value, CultureInfo cultureInfo, CompareOptions comparison = CompareOptions.None)
+    /// <summary>
+    /// Checks if <paramref name="str"/> contains <paramref name="search"/>, ignoring case.
+    /// Diacritics are also ignored unless the search string itself contains diacritics.
+    /// </summary>
+    public static bool ContainsDiacriticMatch(this string str, string search)
     {
-        return cultureInfo.CompareInfo.IndexOf(str, value, comparison) >= 0;
+        var options = DiacriticMatchOptions(search);
+        return CultureInfo.InvariantCulture.CompareInfo.Contains(str, search, options);
     }
 
     /// <summary>
-    /// searches a string for a match ignoring diacritics, but only when the search string does not contain diacritics
+    /// Checks if <paramref name="str"/> starts with <paramref name="search"/>, ignoring case.
+    /// Diacritics are also ignored unless the search string itself contains diacritics.
     /// </summary>
-    /// <param name="str">source of the search</param>
-    /// <param name="search">string to search for</param>
-    public static bool ContainsDiacriticMatch(this string str, string search)
+    public static bool StartsWithDiacriticMatch(this string str, string search)
     {
-        if (ContainsDiacritic(search))
-        {
-            return Contains(str, search, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase);
-        }
+        var options = DiacriticMatchOptions(search);
+        return CultureInfo.InvariantCulture.CompareInfo.IsPrefix(str, search, options);
+    }
 
-        return Contains(str,
-            search,
-            CultureInfo.InvariantCulture,
-            CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace);
+    private static bool Contains(this CompareInfo compareInfo, string source, string value, CompareOptions options)
+    {
+        return compareInfo.IndexOf(source, value, options) >= 0;
+    }
+
+    private static CompareOptions DiacriticMatchOptions(string search)
+    {
+        return ContainsDiacritic(search)
+            ? CompareOptions.IgnoreCase
+            : CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace;
     }
 
     public static bool ContainsDiacritic(string value)