Skip to content

Commit 4bafb5f

Browse files
Enkidu93Enkidu93
authored
Add source and target refs to Pretranslations and WordAlignments (#863)
1 parent e195a31 commit 4bafb5f

31 files changed

Lines changed: 374 additions & 82 deletions

File tree

‎src/Echo/src/EchoEngine/TranslationEngineServiceV1.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,8 @@ await _parallelCorpusPreprocessingService.PreprocessAsync(
142142
EngineId = request.EngineId,
143143
CorpusId = corpus.Id,
144144
TextId = row.TextId,
145-
Refs = { row.Refs.Select(r => r.ToString()) },
145+
SourceRefs = { row.SourceRefs.Select(r => r.ToString()) },
146+
TargetRefs = { row.TargetRefs.Select(r => r.ToString()) },
146147
Translation = row.SourceSegment,
147148
SourceTokens = { tokens },
148149
TranslationTokens = { tokens },

‎src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ await _parallelCorpusPreprocessingService.PreprocessAsync(
9595
EngineId = request.EngineId,
9696
CorpusId = corpus.Id,
9797
TextId = row.TextId,
98-
Refs = { row.Refs.Select(r => r.ToString()) },
98+
SourceRefs = { row.SourceRefs.Select(r => r.ToString()) },
99+
TargetRefs = { row.TargetRefs.Select(r => r.ToString()) },
99100
SourceTokens = { row.SourceSegment.Split() },
100101
TargetTokens = { row.TargetSegment.Split() },
101102
Alignment =

‎src/Machine/src/Serval.Machine.Shared/Consumers/TranslationInsertPretranslationsConsumer.cs‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ CancellationToken cancellationToken
3333
EngineId = content,
3434
CorpusId = pretranslation.CorpusId,
3535
TextId = pretranslation.TextId,
36-
Refs = { pretranslation.Refs },
36+
SourceRefs = { pretranslation.SourceRefs },
37+
TargetRefs = { pretranslation.TargetRefs },
3738
Translation = pretranslation.Translation,
3839
SourceTokens = { pretranslation.SourceTokens },
3940
TranslationTokens = { pretranslation.TranslationTokens },
@@ -74,7 +75,8 @@ JsonSerializerOptions options
7475
string corpusId = "",
7576
textId = "",
7677
translation = "";
77-
IReadOnlyList<string> refs = [],
78+
IReadOnlyList<string> sourceRefs = [],
79+
targetRefs = [],
7880
sourceTokens = [],
7981
translationTokens = [];
8082
IReadOnlyList<SIL.Machine.Corpora.AlignedWordPair> alignedWordPairs = [];
@@ -95,7 +97,15 @@ JsonSerializerOptions options
9597
break;
9698
case "refs":
9799
reader.Read();
98-
refs = JsonSerializer.Deserialize<IList<string>>(ref reader, options)!.ToArray();
100+
targetRefs = JsonSerializer.Deserialize<IList<string>>(ref reader, options)!.ToArray();
101+
break;
102+
case "sourceRefs":
103+
reader.Read();
104+
sourceRefs = JsonSerializer.Deserialize<IList<string>>(ref reader, options)!.ToArray();
105+
break;
106+
case "targetRefs":
107+
reader.Read();
108+
targetRefs = JsonSerializer.Deserialize<IList<string>>(ref reader, options)!.ToArray();
99109
break;
100110
case "translation":
101111
reader.Read();
@@ -126,7 +136,8 @@ JsonSerializerOptions options
126136
{
127137
CorpusId = corpusId,
128138
TextId = textId,
129-
Refs = refs,
139+
SourceRefs = sourceRefs,
140+
TargetRefs = targetRefs,
130141
Translation = translation,
131142
Alignment = alignedWordPairs,
132143
SourceTokens = sourceTokens,

‎src/Machine/src/Serval.Machine.Shared/Consumers/WordAlignmentInsertWordAlignmentsConsumer.cs‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,11 @@ CancellationToken cancellationToken
3131
await call.RequestStream.WriteAsync(
3232
new InsertWordAlignmentsRequest
3333
{
34-
EngineId = (string)content,
34+
EngineId = content,
3535
CorpusId = wordAlignment.CorpusId,
3636
TextId = wordAlignment.TextId,
37-
Refs = { wordAlignment.Refs },
37+
SourceRefs = { wordAlignment.SourceRefs },
38+
TargetRefs = { wordAlignment.TargetRefs },
3839
SourceTokens = { wordAlignment.SourceTokens },
3940
TargetTokens = { wordAlignment.TargetTokens },
4041
Alignment = { Map(wordAlignment.Alignment) }
@@ -78,7 +79,8 @@ JsonSerializerOptions options
7879
}
7980
string corpusId = "",
8081
textId = "";
81-
IReadOnlyList<string> refs = [],
82+
IReadOnlyList<string> sourceRefs = [],
83+
targetRefs = [],
8284
sourceTokens = [],
8385
targetTokens = [];
8486
IReadOnlyList<SIL.Machine.Corpora.AlignedWordPair> alignedWordPairs = [];
@@ -99,7 +101,15 @@ JsonSerializerOptions options
99101
break;
100102
case "refs":
101103
reader.Read();
102-
refs = JsonSerializer.Deserialize<IList<string>>(ref reader, options)!.ToArray();
104+
targetRefs = JsonSerializer.Deserialize<IList<string>>(ref reader, options)!.ToArray();
105+
break;
106+
case "sourceRefs":
107+
reader.Read();
108+
sourceRefs = JsonSerializer.Deserialize<IList<string>>(ref reader, options)!.ToArray();
109+
break;
110+
case "targetRefs":
111+
reader.Read();
112+
targetRefs = JsonSerializer.Deserialize<IList<string>>(ref reader, options)!.ToArray();
103113
break;
104114
case "sourceTokens":
105115
reader.Read();
@@ -124,7 +134,8 @@ JsonSerializerOptions options
124134
{
125135
CorpusId = corpusId,
126136
TextId = textId,
127-
Refs = refs,
137+
SourceRefs = sourceRefs,
138+
TargetRefs = targetRefs,
128139
Alignment = alignedWordPairs,
129140
SourceTokens = sourceTokens,
130141
TargetTokens = targetTokens

‎src/Machine/src/Serval.Machine.Shared/Models/Pretranslation.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ public record Pretranslation
44
{
55
public required string CorpusId { get; init; }
66
public required string TextId { get; init; }
7-
public required IReadOnlyList<string> Refs { get; init; }
7+
public required IReadOnlyList<string> SourceRefs { get; init; }
8+
public required IReadOnlyList<string> TargetRefs { get; init; }
89
public required string Translation { get; init; }
910
public IEnumerable<string>? SourceTokens { get; init; }
1011
public IEnumerable<string>? TranslationTokens { get; init; }

‎src/Machine/src/Serval.Machine.Shared/Models/WordAlignment.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ public record WordAlignment
44
{
55
public required string CorpusId { get; init; }
66
public required string TextId { get; init; }
7-
public required IReadOnlyList<string> Refs { get; init; }
7+
public required IReadOnlyList<string> SourceRefs { get; init; }
8+
public required IReadOnlyList<string> TargetRefs { get; init; }
89
public required IReadOnlyList<string> SourceTokens { get; set; }
910
public required IReadOnlyList<string> TargetTokens { get; set; }
1011
public required IReadOnlyList<AlignedWordPair> Alignment { get; set; }

‎src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ await ParallelCorpusPreprocessingService.PreprocessAsync(
7979
pretranslateWriter.WriteString("corpusId", corpus.Id);
8080
pretranslateWriter.WriteString("textId", row.TextId);
8181
pretranslateWriter.WriteStartArray("refs");
82-
foreach (object rowRef in row.Refs)
82+
foreach (object rowRef in row.TargetRefs)
8383
pretranslateWriter.WriteStringValue(rowRef.ToString());
8484
pretranslateWriter.WriteEndArray();
8585
pretranslateWriter.WriteString("translation", row.SourceSegment);

‎src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ await ParallelCorpusPreprocessingService.PreprocessAsync(
7979
wordAlignmentWriter.WriteString("corpusId", corpus.Id);
8080
wordAlignmentWriter.WriteString("textId", row.TextId);
8181
wordAlignmentWriter.WriteStartArray("refs");
82-
foreach (object rowRef in row.Refs)
82+
foreach (object rowRef in row.TargetRefs)
8383
wordAlignmentWriter.WriteStringValue(rowRef.ToString());
8484
wordAlignmentWriter.WriteEndArray();
8585
wordAlignmentWriter.WriteString("source", row.SourceSegment);

‎src/Machine/test/Serval.Machine.Shared.Tests/Consumers/TranslationInsertPretranslationsConsumerTests.cs‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ namespace Serval.Machine.Shared.Consumers;
77
public class TranslationInsertPretranslationsConsumerTests
88
{
99
[Test]
10-
public async Task HandleMessageAsync()
10+
public async Task HandleMessageAsync_Refs()
1111
{
1212
TestEnvironment env = new();
1313

@@ -40,7 +40,51 @@ await JsonSerializer.SerializeAsync(
4040
EngineId = "engine1",
4141
CorpusId = "corpus1",
4242
TextId = "MAT",
43-
Refs = { "MAT 1:1" },
43+
SourceRefs = { },
44+
TargetRefs = { "MAT 1:1" },
45+
Translation = "translation"
46+
},
47+
Arg.Any<CancellationToken>()
48+
);
49+
}
50+
51+
[Test]
52+
public async Task HandleMessageAsync_SourceAndTargetRefs()
53+
{
54+
TestEnvironment env = new();
55+
56+
await using (MemoryStream stream = new())
57+
{
58+
var obj = new JsonObject();
59+
await JsonSerializer.SerializeAsync(
60+
stream,
61+
new JsonArray
62+
{
63+
new JsonObject
64+
{
65+
["corpusId"] = "corpus1",
66+
["textId"] = "MAT",
67+
["sourceRefs"] = new JsonArray { "MAT 1:1" },
68+
["targetRefs"] = new JsonArray { "MAT 1:1" },
69+
["translation"] = "translation"
70+
}
71+
}
72+
);
73+
74+
stream.Seek(0, SeekOrigin.Begin);
75+
await env.Consumer.HandleMessageAsync("engine1", stream);
76+
}
77+
78+
_ = env.Client.Received(1).InsertPretranslations();
79+
_ = env.PretranslationWriter.Received(1)
80+
.WriteAsync(
81+
new InsertPretranslationsRequest
82+
{
83+
EngineId = "engine1",
84+
CorpusId = "corpus1",
85+
TextId = "MAT",
86+
SourceRefs = { "MAT 1:1" },
87+
TargetRefs = { "MAT 1:1" },
4488
Translation = "translation"
4589
},
4690
Arg.Any<CancellationToken>()

‎src/Machine/test/Serval.Machine.Shared.Tests/Consumers/WordAlignmentInsertWordAlignmentsConsumerTests.cs‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ namespace Serval.Machine.Shared.Consumers;
77
public class WordAlignmentInsertWordAlignmentsConsumerTests
88
{
99
[Test]
10-
public async Task HandleMessageAsync()
10+
public async Task HandleMessageAsync_Refs()
1111
{
1212
TestEnvironment env = new();
1313

@@ -49,7 +49,73 @@ await JsonSerializer.SerializeAsync(
4949
EngineId = "engine1",
5050
CorpusId = "corpus1",
5151
TextId = "MAT",
52-
Refs = { "MAT 1:1" },
52+
SourceRefs = { },
53+
TargetRefs = { "MAT 1:1" },
54+
SourceTokens = { "sourceToken1" },
55+
TargetTokens = { "targetToken1" },
56+
Alignment =
57+
{
58+
new WordAlignment.V1.AlignedWordPair()
59+
{
60+
SourceIndex = 0,
61+
TargetIndex = 0,
62+
Score = 1.0
63+
}
64+
}
65+
},
66+
Arg.Any<CancellationToken>()
67+
);
68+
}
69+
70+
[Test]
71+
public async Task HandleMessageAsync_SourceAndTargetRefs()
72+
{
73+
TestEnvironment env = new();
74+
75+
await using (MemoryStream stream = new())
76+
{
77+
await JsonSerializer.SerializeAsync(
78+
stream,
79+
new JsonArray
80+
{
81+
new JsonObject
82+
{
83+
{ "corpusId", "corpus1" },
84+
{ "textId", "MAT" },
85+
{
86+
"sourceRefs",
87+
new JsonArray { "MAT 1:1" }
88+
},
89+
{
90+
"targetRefs",
91+
new JsonArray { "MAT 1:1" }
92+
},
93+
{
94+
"sourceTokens",
95+
new JsonArray { "sourceToken1" }
96+
},
97+
{
98+
"targetTokens",
99+
new JsonArray { "targetToken1" }
100+
},
101+
{ "alignment", "0-0:1.0:1.0" }
102+
}
103+
}
104+
);
105+
stream.Seek(0, SeekOrigin.Begin);
106+
await env.Handler.HandleMessageAsync("engine1", stream);
107+
}
108+
109+
_ = env.Client.Received(1).InsertWordAlignments();
110+
_ = env.WordAlignmentsWriter.Received(1)
111+
.WriteAsync(
112+
new InsertWordAlignmentsRequest
113+
{
114+
EngineId = "engine1",
115+
CorpusId = "corpus1",
116+
TextId = "MAT",
117+
SourceRefs = { "MAT 1:1" },
118+
TargetRefs = { "MAT 1:1" },
53119
SourceTokens = { "sourceToken1" },
54120
TargetTokens = { "targetToken1" },
55121
Alignment =

0 commit comments

Comments
 (0)