Add chapter-level data counts to execution data

src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs

@@ -3,8 +3,11 @@ namespace Serval.Machine.Shared.Models;
 public record BuildExecutionData
 {
     public int? TrainCount { get; init; }
-    public int? PretranslateCount { get; init; }
-    public int? WordAlignCount { get; init; }
+    public int? InferenceCount { get; init; }
+    public bool? IsTrainFilteredByChapter { get; init; }
+    public bool? IsInferenceFilteredByChapter { get; init; }
+    public IReadOnlyDictionary<string, IReadOnlyDictionary<string, int>>? TrainVerseCount { get; init; }
+    public IReadOnlyDictionary<string, IReadOnlyDictionary<string, int>>? InferenceVerseCount { get; init; }
     public IReadOnlyList<string>? Warnings { get; init; }
     public string? EngineSourceLanguageTag { get; init; }
     public string? EngineTargetLanguageTag { get; init; }

src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs

@@ -52,8 +52,7 @@ await PlatformService.UpdateTargetQuoteConventionAsync(
     protected override async Task UpdateBuildExecutionData(
         string engineId,
         string buildId,
-        int trainCount,
-        int pretranslateCount,
+        PreprocessStats stats,
         string sourceLanguageTag,
         string targetLanguageTag,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora,
@@ -63,16 +62,16 @@ CancellationToken cancellationToken
         bool sourceLanguageHasNativeSupport = ResolveLanguageCode(sourceLanguageTag, out string resolvedSourceLanguage);
         bool targetLanguageHasNativeSupport = ResolveLanguageCode(targetLanguageTag, out string resolvedTargetLanguage);
 
-        if (trainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport))
+        if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport))
         {
             throw new InvalidOperationException(
                 $"At least one language code in build {buildId} is unknown to the base model, and the data specified for training was empty. Build canceled."
             );
         }
 
         IReadOnlyList<string> warnings = GetWarnings(
-            trainCount,
-            pretranslateCount,
+            stats.TrainCount,
+            stats.InferenceCount,
             sourceLanguageTag,
             targetLanguageTag,
             parallelCorpora
@@ -92,8 +91,8 @@ CancellationToken cancellationToken
             { "Event", "BuildPreprocess" },
             { "EngineId", engineId },
             { "BuildId", buildId },
-            { "NumTrainRows", trainCount },
-            { "NumPretranslateRows", pretranslateCount },
+            { "NumTrainRows", stats.TrainCount },
+            { "NumPretranslateRows", stats.InferenceCount },
             { "EngineSourceLanguageTag", sourceLanguageTag },
             { "EngineTargetLanguageTag", targetLanguageTag },
             { "SourceLanguageResolved", resolvedSourceLanguage },
@@ -103,8 +102,18 @@ CancellationToken cancellationToken
         Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString());
         var executionData = new BuildExecutionData()
         {
-            TrainCount = trainCount,
-            PretranslateCount = pretranslateCount,
+            TrainCount = stats.TrainCount,
+            InferenceCount = stats.InferenceCount,
+            TrainVerseCount = stats.TrainVerseCount.ToDictionary(
+                kv => kv.Key,
+                kv => (IReadOnlyDictionary<string, int>)kv.Value
+            ),
+            InferenceVerseCount = stats.InferenceVerseCount.ToDictionary(
+                kv => kv.Key,
+                kv => (IReadOnlyDictionary<string, int>)kv.Value
+            ),
+            IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter,
+            IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter,
             Warnings = warnings,
             EngineSourceLanguageTag = sourceLanguageTag,
             EngineTargetLanguageTag = targetLanguageTag,

src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs

@@ -45,18 +45,12 @@ CancellationToken cancellationToken
         if (engine is null)
             throw new OperationCanceledException($"Engine {engineId} does not exist.  Build canceled.");
 
-        (int trainCount, int inferenceCount) = await WriteDataFilesAsync(
-            buildId,
-            data,
-            buildOptions,
-            cancellationToken
-        );
+        PreprocessStats stats = await WriteDataFilesAsync(buildId, data, buildOptions, cancellationToken);
 
         await UpdateBuildExecutionData(
             engineId,
             buildId,
-            trainCount,
-            inferenceCount,
+            stats,
             engine.SourceLanguage,
             engine.TargetLanguage,
             data,
@@ -65,7 +59,7 @@ await UpdateBuildExecutionData(
 
         await UpdateTargetQuoteConventionAsync(engineId, buildId, data, cancellationToken);
 
-        if (inferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false })
+        if (stats.InferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false })
         {
             throw new InvalidOperationException(
                 $"There was no data specified for inferencing in build {buildId}. Build canceled."
@@ -90,8 +84,7 @@ await UpdateBuildExecutionData(
     protected abstract Task UpdateBuildExecutionData(
         string engineId,
         string buildId,
-        int trainCount,
-        int inferenceCount,
+        PreprocessStats stats,
         string sourceLanguageTag,
         string targetLanguageTag,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora,
@@ -105,7 +98,7 @@ protected virtual Task UpdateTargetQuoteConventionAsync(
         CancellationToken cancellationToken
     ) => Task.CompletedTask;
 
-    protected abstract Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync(
+    protected abstract Task<PreprocessStats> WriteDataFilesAsync(
         string buildId,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora,
         string? buildOptions,

src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs

@@ -0,0 +1,11 @@
+namespace Serval.Machine.Shared.Services;
+
+public record PreprocessStats
+{
+    public int TrainCount { get; set; }
+    public int InferenceCount { get; set; }
+    public bool IsTrainFilteredByChapter { get; set; }
+    public bool IsInferenceFilteredByChapter { get; set; }
+    public Dictionary<string, Dictionary<string, int>> TrainVerseCount { get; set; } = [];
+    public Dictionary<string, Dictionary<string, int>> InferenceVerseCount { get; set; } = [];
+}

src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs

@@ -101,7 +101,11 @@ public Task UpdateBuildExecutionDataAsync(
             new ExecutionDataContract
             {
                 TrainCount = executionData.TrainCount,
-                PretranslateCount = executionData.PretranslateCount,
+                PretranslateCount = executionData.InferenceCount,
+                TrainVerseCount = executionData.TrainVerseCount,
+                PretranslateVerseCount = executionData.InferenceVerseCount,
+                IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter,
+                IsPretranslateFilteredByChapter = executionData.IsInferenceFilteredByChapter,
                 Warnings = executionData.Warnings,
                 EngineSourceLanguageTag = executionData.EngineSourceLanguageTag,
                 EngineTargetLanguageTag = executionData.EngineTargetLanguageTag,

src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs

@@ -100,7 +100,11 @@ public Task UpdateBuildExecutionDataAsync(
             new ExecutionDataContract
             {
                 TrainCount = executionData.TrainCount,
-                WordAlignCount = executionData.WordAlignCount,
+                WordAlignCount = executionData.InferenceCount,
+                IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter,
+                IsWordAlignmentFilteredByChapter = executionData.IsInferenceFilteredByChapter,
+                TrainVerseCount = executionData.TrainVerseCount,
+                WordAlignVerseCount = executionData.InferenceVerseCount,
                 Warnings = executionData.Warnings,
                 EngineSourceLanguageTag = executionData.EngineSourceLanguageTag,
                 EngineTargetLanguageTag = executionData.EngineTargetLanguageTag,