Skip to content

Commit 22463b8

Browse files
Enkidu93Enkidu93
committed
Extract preprocessing functions for testing
1 parent e52b0c5 commit 22463b8

4 files changed

Lines changed: 613 additions & 183 deletions

File tree

‎src/Machine/src/Serval.Machine.Shared/Services/PreprocessStats.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ namespace Serval.Machine.Shared.Services;
22

33
public record PreprocessStats
44
{
5-
public required int TrainCount { get; init; }
6-
public required int InferenceCount { get; init; }
7-
public required bool IsTrainFilteredByChapter { get; init; }
8-
public required bool IsInferenceFilteredByChapter { get; init; }
9-
public required Dictionary<string, Dictionary<string, int>> TrainVerseCount { get; init; }
10-
public required Dictionary<string, Dictionary<string, int>> InferenceVerseCount { get; init; }
5+
public int TrainCount { get; set; }
6+
public int InferenceCount { get; set; }
7+
public bool IsTrainFilteredByChapter { get; set; }
8+
public bool IsInferenceFilteredByChapter { get; set; }
9+
public Dictionary<string, Dictionary<string, int>> TrainVerseCount { get; set; } = [];
10+
public Dictionary<string, Dictionary<string, int>> InferenceVerseCount { get; set; } = [];
1111
}

‎src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs‎

Lines changed: 113 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -63,106 +63,33 @@ await SharedFileService.OpenWriteAsync($"builds/{buildId}/train.key-terms.trg.tx
6363
c.InferenceChapters is not null && c.InferenceChapters.Values.Any(chapters => chapters.Count > 0)
6464
)
6565
);
66-
int trainCount = 0;
67-
int pretranslateCount = 0;
68-
Dictionary<string, Dictionary<string, int>> trainVerseCountByChapter = [];
69-
Dictionary<string, Dictionary<string, int>> pretranslateVerseCountByChapter = [];
66+
PreprocessStats preprocessStats = new()
67+
{
68+
IsTrainFilteredByChapter = isTrainFilteredByChapter,
69+
IsInferenceFilteredByChapter = isPretranslationFilteredByChapter,
70+
};
71+
7072
pretranslateWriter.WriteStartArray();
7173
await ParallelCorpusService.PreprocessAsync(
7274
parallelCorpora,
7375
async (row, trainingDataType) =>
74-
{
75-
if (row.SourceSegment.Length > 0 || row.TargetSegment.Length > 0)
76-
{
77-
if (trainingDataType == TrainingDataType.KeyTerm)
78-
{
79-
await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n");
80-
await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n");
81-
}
82-
else
83-
{
84-
await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n");
85-
await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n");
86-
}
87-
}
88-
if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0)
89-
{
90-
trainCount++;
91-
foreach (object? reference in row.SourceRefs)
92-
{
93-
if (reference is not null and ScriptureRef sr)
94-
{
95-
trainVerseCountByChapter.UpdateValue(
96-
sr.Book,
97-
() => [],
98-
chapters =>
99-
{
100-
if (chapters.TryGetValue(sr.Chapter, out int count))
101-
chapters[sr.Chapter] = count + 1;
102-
else
103-
chapters[sr.Chapter] = 1;
104-
return chapters;
105-
}
106-
);
107-
}
108-
}
109-
}
110-
},
76+
await preprocessStats.ProcessTranslationTrainingRowAsync(
77+
row,
78+
trainingDataType,
79+
sourceTrainWriter,
80+
targetTrainWriter,
81+
sourceKeyTermsTrainWriter,
82+
targetKeyTermsTrainWriter
83+
),
11184
async (row, isInTrainingData, corpusId) =>
112-
{
113-
if (row.SourceSegment.Length > 0 && !isInTrainingData)
114-
{
115-
pretranslateWriter.WriteStartObject();
116-
pretranslateWriter.WriteString("corpusId", corpusId);
117-
pretranslateWriter.WriteString("textId", row.TextId);
118-
pretranslateWriter.WriteStartArray("sourceRefs");
119-
foreach (object rowRef in row.SourceRefs)
120-
pretranslateWriter.WriteStringValue(rowRef.ToString());
121-
pretranslateWriter.WriteEndArray();
122-
pretranslateWriter.WriteStartArray("targetRefs");
123-
foreach (object rowRef in row.TargetRefs)
124-
pretranslateWriter.WriteStringValue(rowRef.ToString());
125-
pretranslateWriter.WriteEndArray();
126-
pretranslateWriter.WriteString("translation", row.SourceSegment);
127-
pretranslateWriter.WriteEndObject();
128-
pretranslateCount++;
129-
foreach (object? reference in row.SourceRefs)
130-
{
131-
if (reference is not null and ScriptureRef sr)
132-
{
133-
pretranslateVerseCountByChapter.UpdateValue(
134-
sr.Book,
135-
() => [],
136-
chapters =>
137-
{
138-
if (chapters.TryGetValue(sr.Chapter, out int count))
139-
chapters[sr.Chapter] = count + 1;
140-
else
141-
chapters[sr.Chapter] = 1;
142-
return chapters;
143-
}
144-
);
145-
}
146-
}
147-
}
148-
if (pretranslateWriter.BytesPending > 1024 * 1024)
149-
await pretranslateWriter.FlushAsync();
150-
},
85+
await preprocessStats.ProcessPretranslateRowAsync(row, isInTrainingData, corpusId, pretranslateWriter),
15186
(bool?)buildOptionsObject?["use_key_terms"] ?? true,
15287
ignoreUsfmMarkers: ["rem", "r"]
15388
);
15489

15590
pretranslateWriter.WriteEndArray();
15691

157-
return new PreprocessStats
158-
{
159-
TrainCount = trainCount,
160-
InferenceCount = pretranslateCount,
161-
IsTrainFilteredByChapter = isTrainFilteredByChapter,
162-
IsInferenceFilteredByChapter = isPretranslationFilteredByChapter,
163-
TrainVerseCount = trainVerseCountByChapter,
164-
InferenceVerseCount = pretranslateVerseCountByChapter,
165-
};
92+
return preprocessStats;
16693
}
16794

16895
protected override async Task UpdateBuildExecutionData(
@@ -211,3 +138,100 @@ CancellationToken cancellationToken
211138
await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken);
212139
}
213140
}
141+
142+
public static partial class PreprocessStatsExtensions
143+
{
144+
public static async Task ProcessTranslationTrainingRowAsync(
145+
this PreprocessStats stats,
146+
ParallelRowContract row,
147+
TrainingDataType trainingDataType,
148+
StreamWriter sourceTrainWriter,
149+
StreamWriter targetTrainWriter,
150+
StreamWriter sourceKeyTermsTrainWriter,
151+
StreamWriter targetKeyTermsTrainWriter
152+
)
153+
{
154+
if (row.SourceSegment.Length > 0 || row.TargetSegment.Length > 0)
155+
{
156+
if (trainingDataType == TrainingDataType.KeyTerm)
157+
{
158+
await sourceKeyTermsTrainWriter.WriteAsync($"{row.SourceSegment}\n");
159+
await targetKeyTermsTrainWriter.WriteAsync($"{row.TargetSegment}\n");
160+
}
161+
else
162+
{
163+
await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n");
164+
await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n");
165+
}
166+
}
167+
if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0)
168+
{
169+
stats.TrainCount++;
170+
foreach (object? reference in row.SourceRefs)
171+
{
172+
if (reference is not null and ScriptureRef sr && sr.IsVerse)
173+
{
174+
stats.TrainVerseCount.UpdateValue(
175+
sr.Book,
176+
() => [],
177+
chapters =>
178+
{
179+
if (chapters.TryGetValue(sr.Chapter, out int count))
180+
chapters[sr.Chapter] = count + 1;
181+
else
182+
chapters[sr.Chapter] = 1;
183+
return chapters;
184+
}
185+
);
186+
}
187+
}
188+
}
189+
}
190+
191+
public static async Task ProcessPretranslateRowAsync(
192+
this PreprocessStats stats,
193+
ParallelRowContract row,
194+
bool isInTrainingData,
195+
string corpusId,
196+
Utf8JsonWriter pretranslateWriter
197+
)
198+
{
199+
if (row.SourceSegment.Length > 0 && !isInTrainingData)
200+
{
201+
pretranslateWriter.WriteStartObject();
202+
pretranslateWriter.WriteString("corpusId", corpusId);
203+
pretranslateWriter.WriteString("textId", row.TextId);
204+
pretranslateWriter.WriteStartArray("sourceRefs");
205+
foreach (object rowRef in row.SourceRefs)
206+
pretranslateWriter.WriteStringValue(rowRef.ToString());
207+
pretranslateWriter.WriteEndArray();
208+
pretranslateWriter.WriteStartArray("targetRefs");
209+
foreach (object rowRef in row.TargetRefs)
210+
pretranslateWriter.WriteStringValue(rowRef.ToString());
211+
pretranslateWriter.WriteEndArray();
212+
pretranslateWriter.WriteString("translation", row.SourceSegment);
213+
pretranslateWriter.WriteEndObject();
214+
stats.InferenceCount++;
215+
foreach (object? reference in row.SourceRefs)
216+
{
217+
if (reference is not null and ScriptureRef sr && sr.IsVerse)
218+
{
219+
stats.InferenceVerseCount.UpdateValue(
220+
sr.Book,
221+
() => [],
222+
chapters =>
223+
{
224+
if (chapters.TryGetValue(sr.Chapter, out int count))
225+
chapters[sr.Chapter] = count + 1;
226+
else
227+
chapters[sr.Chapter] = 1;
228+
return chapters;
229+
}
230+
);
231+
}
232+
}
233+
}
234+
if (pretranslateWriter.BytesPending > 1024 * 1024)
235+
await pretranslateWriter.FlushAsync();
236+
}
237+
}

0 commit comments

Comments
 (0)