Properly populate parallel corpus analyses; add migration (untested)

Author: Enkidu93

src/Serval/src/Serval.Client/Client.g.cs

@@ -10791,9 +10791,8 @@ public partial class TranslationBuild
         [System.Obsolete]
         public System.Collections.Generic.IList<ParallelCorpusAnalysis>? Analysis { get; set; } = default!;
 
-        [Newtonsoft.Json.JsonProperty("targetQuoteConvention", Required = Newtonsoft.Json.Required.Always)]
-        [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
-        public string TargetQuoteConvention { get; set; } = default!;
+        [Newtonsoft.Json.JsonProperty("targetQuoteConvention", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public string? TargetQuoteConvention { get; set; } = default!;
 
         [Newtonsoft.Json.JsonProperty("canDenormalizeQuotes", Required = Newtonsoft.Json.Required.Always)]
         public bool CanDenormalizeQuotes { get; set; } = default!;

src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs

@@ -61,6 +61,21 @@ await c.Aggregate()
                     .AppendStage<BsonDocument>(new BsonDocument("$unset", "engine"))
                     .Merge(c, new MergeStageOptions<Build> { WhenMatched = MergeStageWhenMatched.Replace })
                     .ToListAsync();
+
+                //migrate by adding TargetQuoteConvention field populated from deprecated analysis field
+                // await c.Aggregate()
+                //     .Match(Builders<Build>.Filter.Exists(b => b.TargetQuoteConvention, false))
+                //     .Match(Builders<Build>.Filter.Exists("analysis"))
+                //     .Unwind("analysis")
+                //     .Match(Builders<BsonDocument>.Filter.Ne("$analysis.targetQuoteConvention", ""))
+                //     .AppendStage<BsonDocument>(
+                //         new BsonDocument(
+                //             "$set",
+                //             new BsonDocument("targetQuoteConvention", "$analysis.targetQuoteConvention")
+                //         )
+                //     )
+                //     .Merge(c, new MergeStageOptions<Build> { WhenMatched = MergeStageWhenMatched.Replace })
+                //     .ToListAsync();
             }
         );
         configurator.AddRepository<Pretranslation>(

src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs

@@ -39,6 +39,6 @@ public record TranslationBuildDto
 
     [Obsolete]
     public IReadOnlyList<ParallelCorpusAnalysisDto>? Analysis { get; init; }
-    public required string TargetQuoteConvention { get; init; }
-    public required bool CanDenormalizeQuotes { get; init; }
+    public string? TargetQuoteConvention { get; init; }
+    public bool CanDenormalizeQuotes { get; init; }
 }

src/Serval/src/Serval.Translation/Controllers/TranslationBuildsController.cs

@@ -6,10 +6,12 @@ namespace Serval.Translation.Controllers;
 public class TranslationBuildsController(
     IAuthorizationService authService,
     IBuildService buildService,
-    IUrlService urlService
+    IUrlService urlService,
+    IEngineService engineService
 ) : TranslationControllerBase(authService, urlService)
 {
     private readonly IBuildService _buildService = buildService;
+    private readonly IEngineService _engineService = engineService;
 
     /// <summary>
     /// Get all builds for your translation engines that are created after the specified date.
@@ -31,13 +33,17 @@ public async Task<IEnumerable<TranslationBuildDto>> GetAllBuildsCreatedAfterAsyn
         CancellationToken cancellationToken
     )
     {
+        IEnumerable<Build> builds;
         if (createdAfter is null)
         {
-            return (await _buildService.GetAllAsync(Owner, cancellationToken)).Select(Map);
+            builds = await _buildService.GetAllAsync(Owner, cancellationToken);
         }
         else
         {
-            return (await _buildService.GetAllCreatedAfterAsync(Owner, createdAfter, cancellationToken)).Select(Map);
+            builds = await _buildService.GetAllCreatedAfterAsync(Owner, createdAfter, cancellationToken);
         }
+        return await Task.WhenAll(
+            builds.Select(async b => Map(b, await _engineService.GetAsync(b.EngineRef, cancellationToken)))
+        );
     }
 }

src/Serval/src/Serval.Translation/Controllers/TranslationControllerBase.cs

@@ -7,17 +7,10 @@ public abstract class TranslationControllerBase(IAuthorizationService authServic
 {
     private readonly IUrlService _urlService = urlService;
 
-    protected TranslationBuildDto Map(Build source)
+    protected TranslationBuildDto Map(Build source, Engine engine)
     {
-        string targetQuoteConvention = "";
-        if (source.TargetQuoteConvention != null)
-        {
-            targetQuoteConvention = source.TargetQuoteConvention;
-        }
-        else if (source.Analysis?.Any(a => a.TargetQuoteConvention != "") ?? false)
-        {
-            targetQuoteConvention = source.Analysis.First(a => a.TargetQuoteConvention != "").TargetQuoteConvention;
-        }
+        string targetQuoteConvention = source.TargetQuoteConvention ?? "";
+
         return new TranslationBuildDto
         {
             Id = source.Id,
@@ -45,7 +38,7 @@ protected TranslationBuildDto Map(Build source)
             DeploymentVersion = source.DeploymentVersion,
             ExecutionData = Map(source.ExecutionData),
             Phases = source.Phases?.Select(Map).ToList(),
-            Analysis = source.Analysis?.Select(a => Map(a, targetQuoteConvention)).ToList(),
+            Analysis = Map(engine, targetQuoteConvention).ToList(),
             TargetQuoteConvention = targetQuoteConvention,
             CanDenormalizeQuotes = targetQuoteConvention != ""
         };
@@ -133,14 +126,14 @@ private static PhaseDto Map(BuildPhase source) =>
             Started = source.Started,
         };
 
-    private static ParallelCorpusAnalysisDto Map(ParallelCorpusAnalysis source, string targetQuoteConvention) =>
-        new ParallelCorpusAnalysisDto
+    private static IEnumerable<ParallelCorpusAnalysisDto> Map(Engine engine, string targetQuoteConvention) =>
+        engine.ParallelCorpora.Select(pc => new ParallelCorpusAnalysisDto
         {
-            ParallelCorpusRef = source.ParallelCorpusRef,
+            ParallelCorpusRef = pc.Id,
             TargetQuoteConvention = targetQuoteConvention,
             SourceQuoteConvention = "ignore",
             CanDenormalizeQuotes = targetQuoteConvention != ""
-        };
+        });
 
     private static ExecutionDataDto Map(ExecutionData source) =>
         new ExecutionDataDto

src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs

@@ -1259,7 +1259,13 @@ CancellationToken cancellationToken
     )
     {
         await AuthorizeAsync(id, cancellationToken);
-        return Ok((await _buildService.GetAllAsync(Owner, id, cancellationToken)).Select(Map));
+        return Ok(
+            await Task.WhenAll(
+                (await _buildService.GetAllAsync(Owner, id, cancellationToken)).Select(async b =>
+                    Map(b, await _engineService.GetAsync(b.EngineRef, cancellationToken))
+                )
+            )
+        );
     }
 
     /// <summary>
@@ -1314,13 +1320,16 @@ CancellationToken cancellationToken
             {
                 EntityChangeType.None => StatusCode(StatusCodes.Status408RequestTimeout),
                 EntityChangeType.Delete => NotFound(),
-                _ => Ok(Map(change.Entity!)),
+                _
+                    => Ok(
+                        Map(change.Entity!, await _engineService.GetAsync(change.Entity!.EngineRef, cancellationToken))
+                    ),
             };
         }
         else
         {
             Build build = await _buildService.GetAsync(buildId, cancellationToken);
-            return Ok(Map(build));
+            return Ok(Map(build, await _engineService.GetAsync(build.EngineRef, cancellationToken)));
         }
     }
 
@@ -1394,7 +1403,7 @@ CancellationToken cancellationToken
         if (!await _engineService.StartBuildAsync(build, cancellationToken))
             return Conflict();
 
-        TranslationBuildDto dto = Map(build);
+        TranslationBuildDto dto = Map(build, await _engineService.GetAsync(build.EngineRef, cancellationToken));
         return Created(dto.Url, dto);
     }
 
@@ -1445,7 +1454,10 @@ CancellationToken cancellationToken
             {
                 EntityChangeType.None => StatusCode(StatusCodes.Status408RequestTimeout),
                 EntityChangeType.Delete => NoContent(),
-                _ => Ok(Map(change.Entity!)),
+                _
+                    => Ok(
+                        Map(change.Entity!, await _engineService.GetAsync(change.Entity!.EngineRef, cancellationToken))
+                    ),
             };
         }
         else
@@ -1454,7 +1466,7 @@ CancellationToken cancellationToken
             if (build == null)
                 return NoContent();
 
-            return Ok(Map(build));
+            return Ok(Map(build, await _engineService.GetAsync(build.EngineRef, cancellationToken)));
         }
     }
 
@@ -1490,7 +1502,7 @@ CancellationToken cancellationToken
         Build? build = await _engineService.CancelBuildAsync(id, cancellationToken);
         if (build is null)
             return NoContent();
-        return Ok(Map(build));
+        return Ok(Map(build, await _engineService.GetAsync(build.EngineRef, cancellationToken)));
     }
 
     /// <summary>

src/Serval/src/Serval.Translation/Models/Build.cs

@@ -22,6 +22,5 @@ public record Build : IOwnedEntity
     public DateTime? DateStarted { get; set; }
     public DateTime? DateCompleted { get; set; }
     public IReadOnlyList<BuildPhase>? Phases { get; init; }
-    public IReadOnlyList<ParallelCorpusAnalysis>? Analysis { get; init; }
     public string? TargetQuoteConvention { get; init; }
 }

src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs

@@ -232,7 +232,7 @@ public async Task UpdateTargetQuoteConventionAsync_NoEngine()
 
         build = await env.Builds.GetAsync(c => c.Id == build.Id);
 
-        Assert.That(build?.Analysis, Is.Null);
+        Assert.That(build?.TargetQuoteConvention, Is.EqualTo(""));
     }
 
     [Test]

src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs

@@ -524,14 +524,6 @@ public TestEnvironment()
                         EngineRef = "engine1",
                         Owner = "owner1",
                         DateFinished = DateTime.UnixEpoch,
-                        Analysis =
-                        [
-                            new ParallelCorpusAnalysis()
-                            {
-                                ParallelCorpusRef = "corpus1",
-                                TargetQuoteConvention = "standard_english"
-                            }
-                        ],
                         TargetQuoteConvention = "standard_english"
                     },
                     new()
@@ -540,14 +532,6 @@ public TestEnvironment()
                         EngineRef = "parallel_engine1",
                         Owner = "owner1",
                         DateFinished = DateTime.UnixEpoch,
-                        Analysis =
-                        [
-                            new ParallelCorpusAnalysis()
-                            {
-                                ParallelCorpusRef = "parallel_corpus1",
-                                TargetQuoteConvention = "standard_english"
-                            }
-                        ],
                         TargetQuoteConvention = "standard_english"
                     }
                 ]