Do not write any escaped surrogate pairs to inference JSONs (#888)

Author: Enkidu93

src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs

@@ -19,12 +19,13 @@ IOptionsMonitor<BuildJobOptions> options
     )
     where TEngine : ITrainingEngine
 {
-    // Using UnsafeRelaxedJsonEscaping to avoid escaping surrogate pairs which can result in invalid UTF-8.
+    // Using JavaScriptEncoder.Create(UnicodeRanges.All) to avoid escaping surrogate pairs
+    // (including those outside of the BMP) which can result in invalid UTF-8.
     // This is safe since the data written by this writer is only read internally and only as UTF-8 encoded JSON.
     protected static readonly JsonWriterOptions InferenceWriterOptions = new()
     {
         Indented = true,
-        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
     };
 
     internal BuildJobRunnerType TrainJobRunnerType { get; init; } = BuildJobRunnerType.ClearML;

src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs

@@ -12,12 +12,13 @@ public class SmtTransferTrainBuildJob(
     ITransferEngineFactory transferEngineFactory
 ) : HangfireBuildJob<TranslationEngine>(platformService, engines, dataAccessContext, buildJobService, logger)
 {
-    // Using UnsafeRelaxedJsonEscaping to avoid escaping surrogate pairs which can result in invalid UTF-8.
+    // Using JavaScriptEncoder.Create(UnicodeRanges.All) to avoid escaping surrogate pairs
+    // (including those outside of the BMP) which can result in invalid UTF-8.
     // This is safe since the data written by this writer is only read internally and only as UTF-8 encoded JSON.
-    private static readonly JsonWriterOptions PretranslateWriterOptions = new()
+    protected static readonly JsonWriterOptions PretranslateWriterOptions = new()
     {
         Indented = true,
-        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
     };
     private static readonly JsonSerializerOptions JsonSerializerOptions = new()
     {

src/Machine/src/Serval.Machine.Shared/Services/StatisticalTrainBuildJob.cs

@@ -10,12 +10,13 @@ public class StatisticalTrainBuildJob(
     IWordAlignmentModelFactory wordAlignmentModelFactory
 ) : HangfireBuildJob<WordAlignmentEngine>(platformService, engines, dataAccessContext, buildJobService, logger)
 {
-    // Using UnsafeRelaxedJsonEscaping to avoid escaping surrogate pairs which can result in invalid UTF-8.
+    // Using JavaScriptEncoder.Create(UnicodeRanges.All) to avoid escaping surrogate pairs
+    // (including those outside of the BMP) which can result in invalid UTF-8.
     // This is safe since the data written by this writer is only read internally and only as UTF-8 encoded JSON.
-    private static readonly JsonWriterOptions WordAlignmentWriterOptions = new()
+    protected static readonly JsonWriterOptions WordAlignmentWriterOptions = new()
     {
         Indented = true,
-        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
     };
     private static readonly JsonSerializerOptions JsonSerializerOptions = new()
     {

src/Machine/src/Serval.Machine.Shared/Usings.cs

@@ -18,6 +18,7 @@
 global using System.Text.Json;
 global using System.Text.Json.Nodes;
 global using System.Text.Json.Serialization;
+global using System.Text.Unicode;
 global using Amazon;
 global using Amazon.Runtime;
 global using Amazon.S3;