diff --git a/Taskfile.yml b/Taskfile.yml index a51fcbf4f3..7fb944ce14 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -147,3 +147,8 @@ tasks: aliases: [android-install-emulator-dev] desc: Install the "Dev" flavor on a running x86_64 emulator without building deps: [fw-lite:install-maui-android-emulator-dev] + + fw-lite-android-release-dev: + aliases: [android-release-dev] + desc: Build a Release "Dev" APK and adb-install it on the connected USB device + deps: [fw-lite:install-maui-android-release-dev] diff --git a/backend/Directory.Packages.props b/backend/Directory.Packages.props index 8d18832745..611650c32d 100644 --- a/backend/Directory.Packages.props +++ b/backend/Directory.Packages.props @@ -12,11 +12,14 @@ + - - + + @@ -25,38 +28,38 @@ - - + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + @@ -70,16 +73,16 @@ - - + + - - - + + + - - - + + + @@ -119,9 +122,9 @@ - - - + + + diff --git a/backend/FwLite/FwLiteMaui/FwLiteMaui.csproj b/backend/FwLite/FwLiteMaui/FwLiteMaui.csproj index cfeae4b9ed..923a3053a8 100644 --- a/backend/FwLite/FwLiteMaui/FwLiteMaui.csproj +++ b/backend/FwLite/FwLiteMaui/FwLiteMaui.csproj @@ -60,6 +60,14 @@ $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) + + + + + + + + @@ -100,4 +108,60 @@ --> + + + + <_Linq2DbEfCoreEffectiveVersion>$([System.Text.RegularExpressions.Regex]::Match($([System.IO.File]::ReadAllText('$(MSBuildThisFileDirectory)..\..\Directory.Packages.props')), 'linq2db\.EntityFrameworkCore[^>]*Version="([^"]+)"').Groups[1].Value) + + + + + + + + + + + + <_Linq2DbStagedAssemblies Include="$(IntermediateOutputPath)**\linq2db.EntityFrameworkCore.dll" /> + + + + + + <_Linq2DbPatcherDll>$(MSBuildThisFileDirectory)build\Linq2DbCctorPatcher\bin\$(Configuration)\net10.0\Linq2DbCctorPatcher.dll + + + + + diff --git a/backend/FwLite/FwLiteMaui/build/Linq2DbCctorPatcher/Linq2DbCctorPatcher.csproj b/backend/FwLite/FwLiteMaui/build/Linq2DbCctorPatcher/Linq2DbCctorPatcher.csproj new file mode 100644 index 0000000000..7c1483ed72 --- /dev/null +++ b/backend/FwLite/FwLiteMaui/build/Linq2DbCctorPatcher/Linq2DbCctorPatcher.csproj @@ -0,0 +1,18 @@ + + + Exe + net10.0 + enable + enable + + false + + false + + false + + + + + diff --git a/backend/FwLite/FwLiteMaui/build/Linq2DbCctorPatcher/Program.cs b/backend/FwLite/FwLiteMaui/build/Linq2DbCctorPatcher/Program.cs new file mode 100644 index 0000000000..03e1b416e4 --- /dev/null +++ b/backend/FwLite/FwLiteMaui/build/Linq2DbCctorPatcher/Program.cs @@ -0,0 +1,96 @@ +// Cecil-patches the broken cctor on LinqToDB.EntityFrameworkCore.EFCoreMetadataReader+SqlTransparentExpression +// (and replaces Quote() with a loud throw). See https://github.com/sillsdev/languageforge-lexbox/issues/2291 +// for the background, kill-switch checklist, and upstream PR link. +using Mono.Cecil; +using Mono.Cecil.Cil; + +if (args.Length < 1) +{ + Console.Error.WriteLine("usage: Linq2DbCctorPatcher "); + return 1; +} + +var dllPath = args[0]; +if (!File.Exists(dllPath)) +{ + Console.Error.WriteLine($"File not found: {dllPath}"); + return 2; +} + +var markerPath = dllPath + ".cctor-patched"; +if (File.Exists(markerPath) && File.GetLastWriteTimeUtc(markerPath) >= File.GetLastWriteTimeUtc(dllPath)) +{ + Console.WriteLine($"Already patched: {dllPath}"); + return 0; +} + +// Structural guards: if upstream restructures any of these, the build must +// break loudly. We do NOT skip-on-mismatch — that would silently ship an +// unprotected dll. Bumping the package without re-checking this code is +// already gated by the MSBuild version pin in FwLiteMaui.csproj, but these +// guards are belt-and-braces for the case where someone widens the pin +// without auditing the IL shape. +static int Fail(string message) +{ + Console.Error.WriteLine("Linq2DbCctorPatcher: " + message); + Console.Error.WriteLine( + "linq2db.EntityFrameworkCore structure changed; patcher needs review. " + + "See https://github.com/sillsdev/languageforge-lexbox/issues/2291."); + return 3; +} + +using (var asm = AssemblyDefinition.ReadAssembly(dllPath, new ReaderParameters { ReadWrite = true })) +{ + var outer = asm.MainModule.GetType("LinqToDB.EntityFrameworkCore.EFCoreMetadataReader"); + if (outer is null) + return Fail("EFCoreMetadataReader type not found."); + + var nested = outer.NestedTypes.FirstOrDefault(t => t.Name == "SqlTransparentExpression"); + if (nested is null) + return Fail("SqlTransparentExpression nested type not found inside EFCoreMetadataReader."); + + var cctor = nested.Methods.FirstOrDefault(m => m.IsConstructor && m.IsStatic); + if (cctor is null || !cctor.HasBody) + return Fail("SqlTransparentExpression .cctor not found (or has no body)."); + + // Sanity-check the cctor shape: at least one stsfld targeting the _ctor field. + // If upstream renames _ctor or restructures the field init, we want to know. + var storesCtorField = cctor.Body.Instructions.Any(ins => + ins.OpCode == OpCodes.Stsfld + && ins.Operand is FieldReference fr + && fr.Name == "_ctor" + && fr.DeclaringType.FullName == nested.FullName); + if (!storesCtorField) + return Fail("SqlTransparentExpression .cctor no longer contains a stsfld for the _ctor field; IL shape changed."); + + var quote = nested.Methods.FirstOrDefault(m => m.Name == "Quote" && m.Parameters.Count == 0); + if (quote is null || !quote.HasBody) + return Fail("SqlTransparentExpression.Quote() not found (or has no body)."); + + ReplaceBodyWith(cctor, Instruction.Create(OpCodes.Ret)); + Console.WriteLine("Stubbed SqlTransparentExpression .cctor to no-op ret"); + + // Replace Quote() with `throw new NotImplementedException();` so anything that + // somehow reaches it fails loud rather than NRE'ing on the now-null _ctor field. + var nieCtor = asm.MainModule.ImportReference( + typeof(NotImplementedException).GetConstructor(Type.EmptyTypes)!); + ReplaceBodyWith(quote, + Instruction.Create(OpCodes.Newobj, nieCtor), + Instruction.Create(OpCodes.Throw)); + Console.WriteLine("Replaced SqlTransparentExpression.Quote() with throw NotImplementedException"); + + asm.Write(); +} + +File.WriteAllText(markerPath, DateTime.UtcNow.ToString("O")); +Console.WriteLine($"Patched {dllPath}"); +return 0; + +static void ReplaceBodyWith(MethodDefinition method, params Instruction[] instructions) +{ + method.Body.Instructions.Clear(); + method.Body.ExceptionHandlers.Clear(); + method.Body.Variables.Clear(); + var il = method.Body.GetILProcessor(); + foreach (var ins in instructions) il.Append(ins); +} diff --git a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt index 8773cd85a0..82d3909af3 100644 --- a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt @@ -10,7 +10,6 @@ Keys: Id PK Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -33,7 +32,6 @@ Keys: Id PK Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -71,7 +69,6 @@ Annotations: Relational:Filter: ComponentSenseId IS NOT NULL Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -93,7 +90,6 @@ Indexes: SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -129,7 +125,6 @@ Indexes: SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -171,7 +166,6 @@ Indexes: SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -203,7 +197,6 @@ SenseId SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -236,7 +229,6 @@ Kind Unique SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -259,7 +251,6 @@ Indexes: SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -281,7 +272,6 @@ Indexes: SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -305,7 +295,6 @@ Indexes: SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -331,7 +320,7 @@ SnapshotId (no field, Guid?) Shadow FK Index Navigations: ExampleSentences (List) Collection ToDependent ExampleSentence - PartOfSpeech (PartOfSpeech) ToPrincipal PartOfSpeech + PartOfSpeech (PartOfSpeech) Optional ToPrincipal PartOfSpeech Keys: Id PK Foreign keys: @@ -343,7 +332,6 @@ PartOfSpeechId SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -372,7 +360,6 @@ SnapshotId Unique WsId, Type Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -404,7 +391,6 @@ Keys: Id PK Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -424,7 +410,6 @@ Foreign keys: ChangeEntity {'CommitId'} -> Commit {'Id'} Required Cascade ToDependent: ChangeEntities Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -446,7 +431,7 @@ ElementType: Element type: Guid Required TypeName (string) Required Navigations: - Commit (Commit) ToPrincipal Commit Inverse: Snapshots + Commit (Commit) Required ToPrincipal Commit Inverse: Snapshots Keys: Id PK Foreign keys: @@ -455,7 +440,6 @@ EntityId CommitId, EntityId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -469,7 +453,6 @@ Keys: Id PK Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -489,7 +472,6 @@ Indexes: SnapshotId Unique Annotations: - DiscriminatorProperty: Relational:FunctionName: Relational:Schema: Relational:SqlQuery: @@ -497,4 +479,4 @@ Relational:ViewName: Relational:ViewSchema: Annotations: - ProductVersion: 9.0.16 \ No newline at end of file + ProductVersion: 10.0.8 \ No newline at end of file diff --git a/backend/FwLite/LcmCrdt/Changes/CreateExampleSentenceChange.cs b/backend/FwLite/LcmCrdt/Changes/CreateExampleSentenceChange.cs index 8d9dbebd73..e82b7ef49f 100644 --- a/backend/FwLite/LcmCrdt/Changes/CreateExampleSentenceChange.cs +++ b/backend/FwLite/LcmCrdt/Changes/CreateExampleSentenceChange.cs @@ -1,6 +1,5 @@ using System.ComponentModel; using System.Text.Json.Serialization; -using LinqToDB.Common; using SIL.Harmony; using SIL.Harmony.Changes; using SIL.Harmony.Core; @@ -39,7 +38,7 @@ private CreateExampleSentenceChange(Guid entityId, Guid senseId) : base(entityId public override async ValueTask NewEntity(Commit commit, IChangeContext context) { #pragma warning disable CS0618 // Type or member is obsolete - var translations = Translations ?? (!Translation.IsNullOrEmpty() + var translations = Translations ?? (Translation is { Count: > 0 } ? [MiniLcm.Models.Translation.FromMultiString(Translation)] : []); #pragma warning restore CS0618 // Type or member is obsolete diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index 5bd78128d0..00efaa550e 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -10,6 +10,7 @@ using LcmCrdt.MediaServer; using LcmCrdt.Objects; using LinqToDB; +using LinqToDB.Async; using LinqToDB.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; diff --git a/backend/FwLite/LcmCrdt/Data/EntryQueryHelpers.cs b/backend/FwLite/LcmCrdt/Data/EntryQueryHelpers.cs index 37eb983a7e..f08eda6c3e 100644 --- a/backend/FwLite/LcmCrdt/Data/EntryQueryHelpers.cs +++ b/backend/FwLite/LcmCrdt/Data/EntryQueryHelpers.cs @@ -51,7 +51,7 @@ public static bool SearchHeadwords(this Entry e, string? leading, string? traili Json.QueryValues(e.CitationForm).Any( v => SqlHelpers.ContainsIgnoreCaseAccents(v, query)) || Json.QueryEntries(e.LexemeForm).Any(kv => - string.IsNullOrEmpty((Json.Value(e.CitationForm, ms => ms[kv.Key]) ?? "").Trim()) && + string.IsNullOrEmpty((Json.At(e.CitationForm, kv.Key) ?? "").Trim()) && SqlHelpers.ContainsIgnoreCaseAccents((leading ?? "") + (kv.Value ?? "").Trim() + (trailing ?? ""), query)); } diff --git a/backend/FwLite/LcmCrdt/Data/MiniLcmRepository.cs b/backend/FwLite/LcmCrdt/Data/MiniLcmRepository.cs index c1a8cf690a..8b3273b6da 100644 --- a/backend/FwLite/LcmCrdt/Data/MiniLcmRepository.cs +++ b/backend/FwLite/LcmCrdt/Data/MiniLcmRepository.cs @@ -4,6 +4,7 @@ using LcmCrdt.FullTextSearch; using LcmCrdt.Utils; using LinqToDB; +using LinqToDB.Async; using LinqToDB.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; diff --git a/backend/FwLite/LcmCrdt/EntryFilterMapProvider.cs b/backend/FwLite/LcmCrdt/EntryFilterMapProvider.cs index 80b94417da..1caeb2bf0d 100644 --- a/backend/FwLite/LcmCrdt/EntryFilterMapProvider.cs +++ b/backend/FwLite/LcmCrdt/EntryFilterMapProvider.cs @@ -7,12 +7,10 @@ public class EntryFilterMapProvider : EntryFilterMapProvider { public override Expression> EntrySensesSemanticDomains => e => e.Senses.Select(s => s.SemanticDomains); public override Expression> EntrySensesSemanticDomainsCode => - //ideally we would use Json.Query(s.SemanticDomains) but Gridify doesn't support that, so we have to configure - //linq2db to rewrite this to that. - e => e.Senses.SelectMany(s => s.SemanticDomains).Select(sd => Json.Value(sd, sd => sd.Code)); + //SemanticDomainRows is the json_each rewrite target + e => e.Senses.SelectMany(s => s.SemanticDomainRows).Select(sd => Json.Value(sd, sd => sd.Code)); public override Func? EntrySensesSemanticDomainsConverter => - //linq2db treats Sense.SemanticDomains as a table, if we use "null" then it'll write the query we want - EntryFilter.NormalizeEmptyToNull; + EntryFilter.NormalizeEmptyToEmptyList; public override Expression> EntrySensesExampleSentences => e => e.Senses.Select(s => s.ExampleSentences); public override Expression> EntrySensesExampleSentencesSentence => (e, ws) => e.Senses.SelectMany(s => s.ExampleSentences).Select(example => Json.Value(example.Sentence, ms => ms[ws])!.GetPlainText()); @@ -32,6 +30,7 @@ public class EntryFilterMapProvider : EntryFilterMapProvider public override Func? EntryComplexFormTypesConverter => EntryFilter.NormalizeEmptyToEmptyList; public override Expression> EntryPublishIn => e => e.PublishIn; public override Expression> EntryPublishInId => - e => e.PublishIn.Select(p => Json.Value(p, p => p.Id.ToString())); - public override Func? EntryPublishInConverter => EntryFilter.NormalizeEmptyToNull; + //PublishInRows is the json_each rewrite target + e => e.PublishInRows.Select(p => Json.Value(p, p => p.Id.ToString())); + public override Func? EntryPublishInConverter => EntryFilter.NormalizeEmptyToEmptyList; } diff --git a/backend/FwLite/LcmCrdt/FullTextSearch/EntrySearchService.cs b/backend/FwLite/LcmCrdt/FullTextSearch/EntrySearchService.cs index 7a33e50f2c..d4a6febd1e 100644 --- a/backend/FwLite/LcmCrdt/FullTextSearch/EntrySearchService.cs +++ b/backend/FwLite/LcmCrdt/FullTextSearch/EntrySearchService.cs @@ -1,6 +1,7 @@ using System.Text; using LcmCrdt.Data; using LinqToDB; +using LinqToDB.Async; using LinqToDB.Data; using LinqToDB.DataProvider.SQLite; using LinqToDB.EntityFrameworkCore; @@ -223,16 +224,13 @@ public async Task UpdateEntrySearchTable(Entry entry) private static async Task InsertOrUpdateEntrySearchRecord(EntrySearchRecord record, ITable table) { - await table.InsertOrUpdateAsync(() => new EntrySearchRecord() - { - Id = record.Id, - Headword = record.Headword, - LexemeForm = record.LexemeForm, - CitationForm = record.CitationForm, - Definition = record.Definition, - Gloss = record.Gloss, - }, - exiting => new EntrySearchRecord() + // EntrySearchRecord is an FTS5 virtual table; SQLite rejects UPSERT (ON CONFLICT) on + // virtual tables, so linq2db v6's InsertOrUpdateAsync — which emits ON CONFLICT — fails. + // v5 fell back to a two-statement UPDATE+INSERT because the provider opted out of upsert + // (https://github.com/linq2db/linq2db/blob/v5.4.1/Source/LinqToDB/Linq/QueryRunner.InsertOrReplace.cs#L251-L298); + // for FTS5 a clean DELETE+INSERT is simpler than emulating the UPDATE-first behavior. + await table.DeleteAsync(e => e.Id == record.Id); + await table.InsertAsync(() => new EntrySearchRecord() { Id = record.Id, Headword = record.Headword, diff --git a/backend/FwLite/LcmCrdt/HistoryService.cs b/backend/FwLite/LcmCrdt/HistoryService.cs index e730abf7eb..a7fc86a344 100644 --- a/backend/FwLite/LcmCrdt/HistoryService.cs +++ b/backend/FwLite/LcmCrdt/HistoryService.cs @@ -7,6 +7,7 @@ using LinqToDB.EntityFrameworkCore; using System.Text.RegularExpressions; using MiniLcm.Exceptions; +using LinqToDB.Async; namespace LcmCrdt; @@ -154,7 +155,7 @@ public async Task LoadChangeContext(Guid commitId, int changeInde } var affectedEntries = await GetAffectedEntryIds(change) - .SelectAwait(async entryId => await GetCurrentOrLatestEntry(entryId)) + .Select(async (Guid entryId, CancellationToken _) => await GetCurrentOrLatestEntry(entryId)) .ToArrayAsync(); return new ChangeContext(change, snapshot, affectedEntries); diff --git a/backend/FwLite/LcmCrdt/Json.cs b/backend/FwLite/LcmCrdt/Json.cs index 71dfe76294..0ba71452ee 100644 --- a/backend/FwLite/LcmCrdt/Json.cs +++ b/backend/FwLite/LcmCrdt/Json.cs @@ -1,9 +1,8 @@ using System.Linq.Expressions; using System.Reflection; using System.Text.Json.Serialization.Metadata; -using LcmCrdt.Changes; using LinqToDB; -using LinqToDB.Common; +using LinqToDB.Internal.SqlQuery; using LinqToDB.Mapping; using LinqToDB.SqlQuery; using SIL.Harmony; @@ -14,7 +13,7 @@ public static class Json { sealed class JsonValuePathBuilder : Sql.IExtensionCallBuilder { - public void Build(Sql.ISqExtensionBuilder builder) + public void Build(Sql.ISqlExtensionBuilder builder) { var propExpression = builder.GetExpression(0); @@ -42,7 +41,7 @@ public void Build(Sql.ISqExtensionBuilder builder) parameters.Insert(0, propExpression); - var valueExpression = (ISqlExpression)new SqlExpression(typeof(string), + var valueExpression = (ISqlExpression)new SqlExpression(new DbDataType(typeof(string)), expressionStr, Precedence.Primary, parameters.ToArray()); @@ -51,7 +50,11 @@ public void Build(Sql.ISqExtensionBuilder builder) if (returnType != typeof(string) && returnType != typeof(RichString))//bypass rich string so it can be used with .GetPlainText() { - valueExpression = PseudoFunctions.MakeTryConvert(new SqlDataType(new DbDataType(returnType)), + valueExpression = new SqlFunction( + new DbDataType(returnType), + PseudoFunctions.TRY_CONVERT, + canBeNull: true, + new SqlDataType(new DbDataType(returnType)), new SqlDataType(new DbDataType(typeof(string), DataType.Text)), valueExpression); } @@ -61,7 +64,7 @@ public void Build(Sql.ISqExtensionBuilder builder) private static void BuildParameterPath(Expression? pathBody, List parameters, - Sql.ISqExtensionBuilder builder) + Sql.ISqlExtensionBuilder builder) { while (pathBody is MemberExpression or MethodCallExpression or UnaryExpression) { @@ -86,6 +89,13 @@ private static void BuildParameterPath(Expression? pathBody, { pathBody = mce.Object ?? mce.Arguments[0]; } + else if (mce.Method.DeclaringType == typeof(Sql) && mce.Method.Name == "Alias") + { + //linq2db 6.x's ExposeExpressionVisitor wraps every [ExpressionMethod] substitution in + //Sql.Alias(real, attr.Alias ?? member.Name) as a column-alias hint; peel it so the path + //walker sees the underlying expression. linq2db 5 did not have this wrap. + pathBody = mce.Arguments[0]; + } else { throw new InvalidOperationException($"Invalid property path for expression {mce}."); @@ -212,6 +222,13 @@ public static string ToString(Guid? guid) return guid?.ToString() ?? ""; } + //Json.Value's path walker can't handle a key captured from an outer json_each row; use At for that. + [Sql.Expression("{0}->>{1}", ServerSideOnly = true)] + public static string? At(MultiString value, string key) + { + throw new NotImplementedException("server-side only"); + } + //maps to a row from json_each internal record JsonEach( [property: Column("value")] T Value, @@ -239,7 +256,7 @@ internal static void ExampleSentenceTranslationModifier(JsonTypeInfo typeInfo) var exampleSentence = (ExampleSentence)obj; if (exampleSentence.Translations.Any()) throw new InvalidOperationException("Cannot set translations when they already exist."); var richString = (RichMultiString?)value; - if (richString.IsNullOrEmpty()) return; + if (richString is null or { Count: 0 }) return; #pragma warning disable CS0618 // Type or member is obsolete exampleSentence.Translations = [Translation.FromMultiString(richString)]; #pragma warning restore CS0618 // Type or member is obsolete diff --git a/backend/FwLite/LcmCrdt/LcmCrdt.csproj b/backend/FwLite/LcmCrdt/LcmCrdt.csproj index 4f9c5b714f..16463784b3 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdt.csproj +++ b/backend/FwLite/LcmCrdt/LcmCrdt.csproj @@ -9,7 +9,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index 9d98c1ebbf..4b0b529cae 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -12,9 +12,9 @@ using LcmCrdt.Objects; using LcmCrdt.RemoteSync; using LinqToDB; -using LinqToDB.AspNet.Logging; using LinqToDB.Data; using LinqToDB.EntityFrameworkCore; +using LinqToDB.Extensions.Logging; using LinqToDB.Mapping; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; @@ -130,16 +130,18 @@ public static void ConfigureDbOptions(IServiceProvider provider, DbContextOption nameof(Commit.HybridDateTime) + "." + nameof(HybridDateTime.DateTime))) .HasAttribute(new ColumnAttribute(nameof(HybridDateTime.Counter), nameof(Commit.HybridDateTime) + "." + nameof(HybridDateTime.Counter))) - //tells linq2db to rewrite Sense.SemanticDomains, into Json.Query(Sense.SemanticDomains) - .Entity().Property(s => s.SemanticDomains).HasAttribute(new ExpressionMethodAttribute(SenseSemanticDomainsExpression())) - .Entity().Property(e => e.PublishIn).HasAttribute(new ExpressionMethodAttribute(EntryPublishInExpression())) + //tells linq2db to rewrite Sense.SemanticDomainRows / Entry.PublishInRows into + //Json.Query(). The rewrite lives on the *Rows shadow accessors + //rather than the real IList columns; see Entry.PublishInRows for why. + .Entity().Property(s => s.SemanticDomainRows).IsExpression(SenseSemanticDomainRowsExpression(), isColumn: false) + .Entity().Property(e => e.PublishInRows).IsExpression(EntryPublishInRowsExpression(), isColumn: false) .Entity().Member(r => r.GetPlainText()).IsExpression(r => Json.GetPlainText(r)) .Entity().Member(g => g.ToString()).IsExpression(g => Json.ToString(g)) .Build(); mappingSchema.SetConvertExpression((WritingSystemId id) => new DataParameter { Value = id.Code, DataType = DataType.Text }); optionsBuilder.AddMappingSchema(mappingSchema); - optionsBuilder.AddCustomOptions(options => options.UseSQLiteMicrosoft()); + optionsBuilder.AddCustomOptions(options => options.UseSQLite()); // Register read-relevant interceptors for LinqToDB var sqliteFunctionInterceptor = new CustomSqliteFunctionInterceptor(); @@ -161,19 +163,16 @@ public static void ConfigureDbOptions(IServiceProvider provider, DbContextOption builder.AddInterceptors(updateSearchTableInterceptor); } - private static Expression>> SenseSemanticDomainsExpression() + private static Expression>> SenseSemanticDomainRowsExpression() { - //using Sql.Property, otherwise if we used `s.SemanticDomains` again it would be recursively rewritten - return s => Json.Query(Sql.Property>(s, nameof(Sense.SemanticDomains))); + return s => Json.Query(s.SemanticDomains); } - private static Expression>> EntryPublishInExpression() + private static Expression>> EntryPublishInRowsExpression() { - //using Sql.Property, otherwise if we used `e.PublishIn` again it would be recursively rewritten - return e => Json.Query(Sql.Property>(e, nameof(Entry.PublishIn))); + return e => Json.Query(e.PublishIn); } - public static void ConfigureCrdt(CrdtConfig config) { config.EnableProjectedTables = true; @@ -263,7 +262,7 @@ public static void ConfigureCrdt(CrdtConfig config) list => JsonSerializer.Serialize(list, (JsonSerializerOptions?)null), json => JsonSerializer.Deserialize(json, (JsonSerializerOptions?)null) ?? Array.Empty()); - var writingSystemArrayConverter = new ValueConverter( + var writingSystemArrayConverter = new Microsoft.EntityFrameworkCore.Storage.ValueConversion.ValueConverter( list => JsonSerializer.Serialize(list, (JsonSerializerOptions?)null), json => json == null ? null : JsonSerializer.Deserialize(json, (JsonSerializerOptions?)null)); builder.Property(v => v.Vernacular) diff --git a/backend/FwLite/LcmCrdt/Objects/DbTranslationDeserializationTarget.cs b/backend/FwLite/LcmCrdt/Objects/DbTranslationDeserializationTarget.cs index a249a5cd10..7651c3b0e9 100644 --- a/backend/FwLite/LcmCrdt/Objects/DbTranslationDeserializationTarget.cs +++ b/backend/FwLite/LcmCrdt/Objects/DbTranslationDeserializationTarget.cs @@ -1,6 +1,5 @@ using System.Text.Json; using System.Text.Json.Serialization; -using LinqToDB.Common; namespace LcmCrdt.Objects; @@ -21,7 +20,7 @@ private class DbTranslationDeserializationTargetConverter: JsonConverter(ref reader, options); - if (translation.IsNullOrEmpty()) return null; + if (translation is null or { Count: 0 }) return null; return new DbTranslationDeserializationTarget(translation); } return null; diff --git a/backend/FwLite/MiniLcm.Tests/FluentAssertGlobalConfig.cs b/backend/FwLite/MiniLcm.Tests/FluentAssertGlobalConfig.cs index 3cfb18764d..c8b247b304 100644 --- a/backend/FwLite/MiniLcm.Tests/FluentAssertGlobalConfig.cs +++ b/backend/FwLite/MiniLcm.Tests/FluentAssertGlobalConfig.cs @@ -16,6 +16,9 @@ public static void Initialize() .ComparingByMembers() .Excluding(m => (m.DeclaringType == typeof(ComplexFormComponent) || m.DeclaringType == typeof(WritingSystem)) && (m.Name == nameof(ComplexFormComponent.Id) || m.Name == nameof(ComplexFormComponent.MaybeId))) + //Shadow query-rewrite targets — domain state lives on the underlying collection. + .Excluding(m => (m.DeclaringType == typeof(Entry) && m.Name == nameof(Entry.PublishInRows)) + || (m.DeclaringType == typeof(Sense) && m.Name == nameof(Sense.SemanticDomainRows))) ); } } diff --git a/backend/FwLite/MiniLcm.Tests/RichMultiStringTests.cs b/backend/FwLite/MiniLcm.Tests/RichMultiStringTests.cs index cfdbfeed0c..59787a3739 100644 --- a/backend/FwLite/MiniLcm.Tests/RichMultiStringTests.cs +++ b/backend/FwLite/MiniLcm.Tests/RichMultiStringTests.cs @@ -295,6 +295,42 @@ public void JsonPatchCanAddRichMultiString() ms["fr"].Should().BeEquivalentTo(new RichString("test", "fr")); } + //emulates older commits in the sync stream where a RichMultiString value was serialized as a plain string + [Fact] + public void JsonPatchCanAddRichMultiStringWhenValueIsString() + { + var ms = new RichMultiString() { { "en", new RichString("existing", "en") } }; + var patch = new JsonPatchDocument(); + patch.Operations.Add(new Operation("add", "/fr", null, "test")); + patch.ApplyTo(ms); + ms.Should().ContainKey("fr"); + ms["fr"].Should().BeEquivalentTo(new RichString("test", "fr")); + } + + //same as above but patching via the parent Entry, so the patch goes through PocoAdapter / + //DictionaryPropertyProxy and hits RichMultiString's IDictionary.Add instead of the typed Add. + [Fact] + public void JsonPatchCanAddRichMultiStringPropertyOnEntityWhenValueIsString() + { + var entry = new Entry(); + var patch = new JsonPatchDocument(); + patch.Operations.Add(new Operation("add", "/Note/en", null, "test")); + patch.ApplyTo(entry); + entry.Note.Should().ContainKey("en"); + entry.Note["en"].Should().BeEquivalentTo(new RichString("test", "en")); + } + + [Fact] + public void JsonPatchCanAddRichMultiStringPropertyOnEntityWithRichStringValue() + { + var entry = new Entry(); + var patch = new JsonPatchDocument(); + patch.Add(e => e.Note["en"], new RichString("test", "en")); + patch.ApplyTo(entry); + entry.Note.Should().ContainKey("en"); + entry.Note["en"].Should().BeEquivalentTo(new RichString("test", "en")); + } + [Fact] public void RichSpanEquality_TrueWhenMatching() { diff --git a/backend/FwLite/MiniLcm/Models/Entry.cs b/backend/FwLite/MiniLcm/Models/Entry.cs index 550080e628..8f65638d89 100644 --- a/backend/FwLite/MiniLcm/Models/Entry.cs +++ b/backend/FwLite/MiniLcm/Models/Entry.cs @@ -1,3 +1,8 @@ +using System.ComponentModel; +using System.ComponentModel.DataAnnotations.Schema; +using System.Text.Json.Serialization; +using MiniLcm.Attributes; + namespace MiniLcm.Models; public record Entry : IObjectWithId @@ -29,6 +34,11 @@ public record Entry : IObjectWithId public virtual List PublishIn { get; set; } = []; + //Server-side query rewrite target — LcmCrdt rewrites this to Json.Query(PublishIn) so + //filter projections (e.g. PublishInRows.Select(...).Any(...)) translate to json_each() SQL. + [MiniLcmInternal, NotMapped, JsonIgnore, EditorBrowsable(EditorBrowsableState.Never)] + public IEnumerable PublishInRows => PublishIn; + public const string UnknownHeadword = "(Unknown)"; public string Headword() diff --git a/backend/FwLite/MiniLcm/Models/RichMultiString.cs b/backend/FwLite/MiniLcm/Models/RichMultiString.cs index 777d0582b3..e5ba5453d5 100644 --- a/backend/FwLite/MiniLcm/Models/RichMultiString.cs +++ b/backend/FwLite/MiniLcm/Models/RichMultiString.cs @@ -38,10 +38,9 @@ public RichMultiString Copy() void IDictionary.Add(object key, object? value) { - var valStr = value as RichString ?? - throw new ArgumentException($"unable to convert value {value?.GetType().Name ?? "null"} to RichString", - nameof(value)); - Add(WritingSystemId.FromUnknown(key), valStr); + var richString = value as RichString ?? + throw new ArgumentException($"unable to convert value {value?.GetType().Name ?? "null"} to RichString", nameof(value)); + Add(WritingSystemId.FromUnknown(key), richString); } public void Add(WritingSystemId key, RichString value) @@ -104,12 +103,11 @@ public RichString this[WritingSystemId key] get => dictionary.TryGetValue(key, out var value) ? value : new RichString([]); set { - // SystemTextJsonPatch's DictionaryTypedPropertyProxy casts to IDictionary - // and may pass null (e.g. when an empty-string RichString deserialized to null). Treat - // that as a remove, mirroring the explicit IDictionary.this[object] setter below. + // value will be null if an empty string was deserialized as a RichString (e.g. from a JsonPatch operation + // routed through DictionaryTypedPropertyProxy). Mirror the IDictionary.this[object] setter and remove the key. if (value is null) { - dictionary.Remove(key); + Remove(key); return; } value.EnsureWs(key); diff --git a/backend/FwLite/MiniLcm/Models/Sense.cs b/backend/FwLite/MiniLcm/Models/Sense.cs index 3c4016f13e..b4938dc867 100644 --- a/backend/FwLite/MiniLcm/Models/Sense.cs +++ b/backend/FwLite/MiniLcm/Models/Sense.cs @@ -1,3 +1,5 @@ +using System.ComponentModel; +using System.ComponentModel.DataAnnotations.Schema; using System.Text.Json; using System.Text.Json.Serialization; using MiniLcm.Attributes; @@ -18,6 +20,10 @@ public class Sense : IObjectWithId, IOrderable public virtual PartOfSpeech? PartOfSpeech { get; set; } = null; public virtual Guid? PartOfSpeechId { get; set; } public virtual IList SemanticDomains { get; set; } = []; + + //Server-side query rewrite target — see Entry.PublishInRows. + [MiniLcmInternal, NotMapped, JsonIgnore, EditorBrowsable(EditorBrowsableState.Never)] + public IEnumerable SemanticDomainRows => SemanticDomains; public virtual List ExampleSentences { get; set; } = []; public Guid[] GetReferences() diff --git a/backend/FwLite/Taskfile.yml b/backend/FwLite/Taskfile.yml index 3879bd15f1..e697c598f0 100644 --- a/backend/FwLite/Taskfile.yml +++ b/backend/FwLite/Taskfile.yml @@ -101,6 +101,21 @@ tasks: - task: install-maui-android vars: { FLAVOR: 'Dev', EXTRA_ARGS: '-p:RuntimeIdentifiers=android-x64 -p:RuntimeIdentifier=android-x64 -p:AdbTarget=-e' } + publish-maui-android-release: + deps: [ ui:build-viewer ] + dir: ./FwLiteMaui + # Produces a signed AAB/APK at bin/Release/net10.0-android/-Signed.{aab,apk}. + # No -t:Run — the SDK's Run uses bundletool, which errors out with multiple devices/emulators. + cmd: dotnet build -f net10.0-android -c Release -t:InstallAndroidDependencies -p:AcceptAndroidSdkLicenses=True -p:FwLiteFlavor={{.FLAVOR}} {{.CLI_ARGS}} + + install-maui-android-release-dev: + desc: Build a Release "Dev" APK and adb-install it on the connected USB device (uses `adb -d`) + dir: ./FwLiteMaui + cmds: + - task: publish-maui-android-release + vars: { FLAVOR: 'Dev' } + - adb -d install -r bin/Release/net10.0-android/org.sil.FwLiteMaui.dev-Signed.apk + build-mini-lcm-sdk: desc: Builds the sdk, a zip with the FwLiteWeb server with a project and config to run locally dir: ./FwLiteWeb diff --git a/backend/LexBoxApi/Services/CrdtCommitService.cs b/backend/LexBoxApi/Services/CrdtCommitService.cs index eed0dcc33e..d4ec51f8c9 100644 --- a/backend/LexBoxApi/Services/CrdtCommitService.cs +++ b/backend/LexBoxApi/Services/CrdtCommitService.cs @@ -1,6 +1,7 @@ using LexCore.Utils; using LexData; using LinqToDB; +using LinqToDB.Async; using LinqToDB.Data; using LinqToDB.EntityFrameworkCore; using LinqToDB.EntityFrameworkCore.Internal; @@ -15,26 +16,18 @@ public async Task AddCommits(Guid projectId, IAsyncEnumerable comm await using var transaction = await dbContext.Database.BeginTransactionAsync(token); var linqToDbContext = dbContext.CreateLinqToDBContext(); await using var tmpTable = await linqToDbContext.CreateTempTableAsync($"tmp_crdt_commit_import_{projectId}__{Guid.NewGuid()}", cancellationToken: token); - await tmpTable.BulkCopyAsync(new BulkCopyOptions{BulkCopyType = BulkCopyType.ProviderSpecific, MaxBatchSize = 10}, commits, token); + //Stamp ProjectId while streaming so the merge below can be a plain column-to-column copy. + //A projection lambda here would let linq2db v6 wrap our Sql.Expr<...>::jsonb cast in the + //EF value-converter (JsonSerializer.Serialize) and fail SQL translation. + var stampedCommits = commits.Select(c => { c.ProjectId = projectId; return c; }); + await tmpTable.BulkCopyAsync(new BulkCopyOptions{BulkCopyType = BulkCopyType.ProviderSpecific, MaxBatchSize = 10}, stampedCommits, token); var commitsTable = linqToDbContext.GetTable(); await commitsTable .Merge() .Using(tmpTable) .OnTargetKey() - .InsertWhenNotMatched(commit => new ServerCommit(commit.Id) - { - Id = commit.Id, - ClientId = commit.ClientId, - HybridDateTime = new HybridDateTime(commit.HybridDateTime.DateTime, commit.HybridDateTime.Counter) - { - DateTime = commit.HybridDateTime.DateTime, Counter = commit.HybridDateTime.Counter - }, - ProjectId = projectId, - Metadata = commit.Metadata, - //without this sql cast the value will be treated as text and fail to insert into the jsonb column - ChangeEntities = Sql.Expr>>($"{commit.ChangeEntities}::jsonb") - }) + .InsertWhenNotMatched() .MergeAsync(token); await transaction.CommitAsync(token); diff --git a/backend/LexData/DataKernel.cs b/backend/LexData/DataKernel.cs index 79f35275c8..c1aa1774a4 100644 --- a/backend/LexData/DataKernel.cs +++ b/backend/LexData/DataKernel.cs @@ -1,7 +1,7 @@ using LexData.Configuration; using LinqToDB; -using LinqToDB.AspNet.Logging; using LinqToDB.EntityFrameworkCore; +using LinqToDB.Extensions.Logging; using LinqToDB.Mapping; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; diff --git a/backend/LexData/LexData.csproj b/backend/LexData/LexData.csproj index 9c5454bf25..54e2e970ea 100644 --- a/backend/LexData/LexData.csproj +++ b/backend/LexData/LexData.csproj @@ -2,8 +2,8 @@ - + all diff --git a/backend/Testing/LexCore/Services/CrdtCommitServiceTests.cs b/backend/Testing/LexCore/Services/CrdtCommitServiceTests.cs index adcdad5bd6..cfd853c6c9 100644 --- a/backend/Testing/LexCore/Services/CrdtCommitServiceTests.cs +++ b/backend/Testing/LexCore/Services/CrdtCommitServiceTests.cs @@ -66,7 +66,6 @@ public async Task CanQueryOldCommits() { var projectId = await _lexBoxDbContext.Projects.Select(p => p.Id).FirstOrDefaultAsync(); var context = _lexBoxDbContext.CreateLinqToDBContext(); - var table = LinqToDB.DataExtensions.GetTable(context); var commitId = Guid.NewGuid(); var changeEntity = new ChangeEntity { @@ -87,21 +86,23 @@ public async Task CanQueryOldCommits() //the old format stored json in json, this is emulating that. changeEntityJson["Change"] = changeEntityJson["Change"]?.ToJsonString(); var jsonPayload = changeEntityJson.ToJsonString(); - var inlineSql = $"'[{jsonPayload}]'::jsonb"; - //insert a new server commit, manually specifying the value for ChangeEntities so it will match the old format. - await LinqToDB.LinqExtensions.InsertAsync(table, () => new ServerCommit(commitId) - { - Id = commitId, - ClientId = Guid.NewGuid(), - HybridDateTime = new HybridDateTime(DateTimeOffset.UtcNow, 0) - { - DateTime = DateTimeOffset.UtcNow, - Counter = 0 - }, - ProjectId = projectId, - Metadata = new CommitMetadata(), - ChangeEntities = LinqToDB.Sql.Expr>>(inlineSql) - }); + //Insert a synthetic old-format commit via raw SQL so we can put pre-serialized + //JSON in ChangeEntities. Linq2Db v6 unconditionally wraps any column assignment + //(including Sql.Expr) in the EF JSON value converter inside an InsertAsync + //projection lambda, so we can't use the typed API for this test case. + var inlinePayload = $"[{jsonPayload}]"; + await LinqToDB.Data.DataContextExtensions.ExecuteAsync( + context, + """ + INSERT INTO "CrdtCommits" + ("Id", "ClientId", "HybridDateTime_DateTime", "HybridDateTime_Counter", "ProjectId", "Metadata", "ChangeEntities") + VALUES (@id, @clientId, @dt, 0, @projectId, '{}'::jsonb, @payload::jsonb) + """, + new LinqToDB.Data.DataParameter("id", commitId, LinqToDB.DataType.Guid), + new LinqToDB.Data.DataParameter("clientId", Guid.NewGuid(), LinqToDB.DataType.Guid), + new LinqToDB.Data.DataParameter("dt", DateTimeOffset.UtcNow, LinqToDB.DataType.DateTimeOffset), + new LinqToDB.Data.DataParameter("projectId", projectId, LinqToDB.DataType.Guid), + new LinqToDB.Data.DataParameter("payload", inlinePayload, LinqToDB.DataType.NVarChar)); var commits = await _lexBoxDbContext.CrdtCommits(projectId).ToArrayAsync(); var actualCommit = commits.Should().ContainSingle(c => c.Id == commitId).Subject; actualCommit.ChangeEntities.Should().BeEquivalentTo([changeEntity], diff --git a/backend/harmony b/backend/harmony index 50b0e2daf0..96a75b26b5 160000 --- a/backend/harmony +++ b/backend/harmony @@ -1 +1 @@ -Subproject commit 50b0e2daf0392082eb6f552372737a6241ed8cd9 +Subproject commit 96a75b26b59bfcde23968c5e9c59674aef59242e