Skip to content

Commit 2253633

Browse files
Js41637Js41637
committed
refactor extract hero convo to be faster, fix non-voiceline sounds not being extracted
1 parent d736f6c commit 2253633

4 files changed

Lines changed: 98 additions & 33 deletions

File tree

‎DataTool/FindLogic/Combo.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Collections.Concurrent;
23
using System.Collections.Generic;
34
using System.Diagnostics;
45
using System.IO;
@@ -16,7 +17,7 @@
1617
namespace DataTool.FindLogic;
1718

1819
public static class Combo {
19-
private static readonly HashSet<ushort> s_unhandledTypes = new HashSet<ushort>();
20+
private static readonly ConcurrentBag<ushort> s_unhandledTypes = new ConcurrentBag<ushort>();
2021

2122
public class ComboInfo {
2223
// keep everything at top level, stops us from doing the same things again.
@@ -1209,7 +1210,8 @@ public static ComboInfo Find(ComboInfo info, ulong guid, Dictionary<ulong, ulong
12091210
break;
12101211
}
12111212
default: {
1212-
if (s_unhandledTypes.Add(guidType)) {
1213+
if (!s_unhandledTypes.Contains(guidType)) {
1214+
s_unhandledTypes.Add(guidType);
12131215
Debugger.Log(0, "DataTool", $"[DataTool.FindLogic.Combo]: Unhandled type: {guidType:X3}\r\n");
12141216
}
12151217

‎DataTool/Helper/IO.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,4 +385,21 @@ public static void CreateDirectorySafe(string? david) {
385385
using var reader = new BinaryReader(stream);
386386
return new teSubtitleThing(reader);
387387
}
388+
389+
/// <summary>
390+
/// Returns the amount of threads to use for parallel processing, taking into account if parallel processing is disabled in the flags, and the amount of processors on the machine.
391+
/// Will always return at least 1.
392+
/// </summary>
393+
public static int GetParallelismAmount(int desiredAmount) {
394+
if (Flags.NoParallelProcessing) {
395+
return 1;
396+
}
397+
398+
int processorCount = Environment.ProcessorCount;
399+
if (desiredAmount <= 0) {
400+
return 1;
401+
}
402+
403+
return Math.Min(desiredAmount, processorCount);
404+
}
388405
}

‎DataTool/ToolFlags.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,5 +63,8 @@ public class ToolFlags : IToolFlags {
6363
[CLIFlag(Default = false, Flag = "debug", Help = "Enable debug logging", Hidden = true, Parser = new[] {"DataTool.Flag.Converter", "CLIFlagBoolean"})]
6464
public bool Debug;
6565

66+
[CLIFlag(Default = false, Flag = "no-parallel", Help = "No parallel processing", Hidden = true, Parser = new[] {"DataTool.Flag.Converter", "CLIFlagBoolean"})]
67+
public bool NoParallelProcessing;
68+
6669
public override bool Validate() => true;
6770
}

‎DataTool/ToolLogic/Extract/ExtractHeroConversations.cs‎

Lines changed: 74 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
1-
using System.Collections.Generic;
1+
using System.Collections.Concurrent;
2+
using System.Collections.Generic;
23
using System.IO;
34
using System.Linq;
5+
using System.Threading.Tasks;
46
using DataTool.DataModels;
57
using DataTool.FindLogic;
68
using DataTool.Flag;
79
using DataTool.Helper;
810
using DataTool.ToolLogic.Util;
11+
using Spectre.Console;
912
using TankLib;
1013
using TankLib.Helpers;
1114
using TankLib.STU.Types;
@@ -36,12 +39,13 @@ public void Parse(ICLIFlags toolFlags) {
3639
Logger.Log("Generating voiceline mappings, this will take a moment...");
3740
GenerateVoicelineMapping();
3841
ProcessConversations(flags, path, parsedTypes);
39-
42+
4043
LogUnknownQueries(parsedTypes);
4144
}
4245

4346
private const string Container = "HeroConvo";
44-
private static readonly Dictionary<ulong, (string heroName, Combo.VoiceLineInstanceInfo voiceLineInstance)> VoicelineHeroMapping = new Dictionary<ulong, (string heroName, Combo.VoiceLineInstanceInfo voiceLineInstance)>();
47+
private readonly ConcurrentDictionary<ulong, bool> SeenVoiceSets = new();
48+
private static readonly ConcurrentDictionary<ulong, (string heroName, Combo.VoiceLineInstanceInfo voiceLineInstance)> VoicelineHeroMapping = new();
4549

4650
private void ProcessConversations(ExtractFlags flags, string basePath, Dictionary<string, ParsedHero> parsedTypes) {
4751
foreach (var conversationGuid in Program.TrackedFiles[0xD0]) {
@@ -93,8 +97,17 @@ private void ProcessConversations(ExtractFlags flags, string basePath, Dictionar
9397
i++;
9498
var (heroName, instance) = VoicelineHeroMapping[voicelineGuid.m_E295B99C];
9599

96-
// todo: hammond could partake in a conversation where he just squeaks in response...
97-
// meaning no voice sound files
100+
if (instance.SoundFiles.Count == 0) {
101+
Logger.Debug($"No sound files found for this voiceline instance {teResourceGUID.AsString(instance.GUIDx06F)}, trying to find them in the STUSound {teResourceGUID.AsString(instance.ExternalSound)}");
102+
var stuSound = GetInstance<STUSound>(instance.ExternalSound);
103+
foreach (var mSoundWemFile in stuSound?.m_C32C2195?.m_soundWEMFiles ?? []) {
104+
if (ExtractHeroVoiceBetter.BadSoundFiles.Contains(mSoundWemFile)) {
105+
continue;
106+
}
107+
108+
instance.SoundFiles.Add(mSoundWemFile);
109+
}
110+
}
98111

99112
foreach (var soundFile in instance.SoundFiles) {
100113
var soundFileGuid = teResourceGUID.AsString(soundFile);
@@ -107,27 +120,71 @@ private void ProcessConversations(ExtractFlags flags, string basePath, Dictionar
107120
}
108121

109122
public void GenerateVoicelineMapping() {
110-
var seenVoiceSets = new HashSet<ulong>();
111-
112123
var heroesDict = Helpers.GetHeroes();
113-
var heroes = heroesDict.Values
124+
var sortedHeroes = heroesDict.Values
114125
.OrderBy(x => !x.IsHero) // sort by hero first
115126
.ThenBy(x => x.GUID.GUID) // then by GUID
116127
.ToArray();
117128

118-
foreach (var hero in heroes) {
119-
var heroStu = hero.STU;
129+
var heroes = sortedHeroes.Where(x => x.IsHero).ToArray();
130+
var npcs = sortedHeroes.Where(x => !x.IsHero).ToArray();
120131

132+
AnsiConsole.Progress()
133+
.AutoRefresh(true)
134+
.AutoClear(true)
135+
.HideCompleted(true)
136+
.Columns(new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn(), new RemainingTimeColumn())
137+
.Start(ctx => {
138+
var task = ctx.AddTask("Generating voiceline mapping", new ProgressTaskSettings {
139+
AutoStart = true,
140+
MaxValue = sortedHeroes.Length
141+
});
142+
143+
GenerateVoiceLineMapping(heroes, task, ctx);
144+
GenerateVoiceLineMapping(npcs, task, ctx);
145+
});
146+
147+
148+
foreach (var guid in Program.TrackedFiles[0x5F]) {
149+
if (SeenVoiceSets.ContainsKey(guid)) {
150+
continue;
151+
}
152+
153+
var voiceSet = GetInstance<STUVoiceSet>(guid);
154+
if (voiceSet == null) continue;
155+
156+
var npcName = $"{IO.GetCleanString(voiceSet.m_269FC4E9)} {IO.GetCleanString(voiceSet.m_C0835C08)}".Trim();
157+
if (string.IsNullOrEmpty(npcName)) {
158+
npcName = IO.GetNullableGUIDName(guid) ?? $"Unknown{teResourceGUID.Index(guid):X}";
159+
}
160+
161+
var info = new Combo.ComboInfo();
162+
FindVoicelinesInVoiceSet(guid, npcName, ref info);
163+
}
164+
}
165+
166+
private void GenerateVoiceLineMapping(HeroVM[] heroes, ProgressTask task, ProgressContext ctx) {
167+
Parallel.ForEach(heroes, new ParallelOptions {
168+
MaxDegreeOfParallelism = IO.GetParallelismAmount(8)
169+
}, hero => {
121170
string heroName = IO.GetValidFilename(hero.Name ?? $"Unknown{teResourceGUID.Index(hero.GUID)}");
122-
Logger.Info($"Generating mapping for {heroName}");
171+
var heroStu = hero.STU;
172+
173+
var heroTask = ctx.AddTask($"Processing {heroName}", new ProgressTaskSettings {
174+
AutoStart = true,
175+
MaxValue = 1
176+
});
123177

124178
Combo.ComboInfo baseInfo = default;
125179
var heroVoiceSetGuid = GetInstance<STUVoiceSetComponent>(heroStu.m_gameplayEntity)?.m_voiceDefinition;
126-
seenVoiceSets.Add(heroVoiceSetGuid);
180+
SeenVoiceSets.TryAdd(heroVoiceSetGuid ?? 0, true);
127181

128182
if (FindVoicelinesInVoiceSet(heroVoiceSetGuid, heroName, ref baseInfo)) {
129183
var skins = new ProgressionUnlocks(heroStu).GetUnlocksOfType(UnlockType.Skin);
184+
heroTask.MaxValue = skins.Count();
185+
130186
foreach (var unlock in skins) {
187+
heroTask.Increment(1);
131188
if (!(unlock.STU is STUUnlock_SkinTheme unlockSkinTheme)) return;
132189
if (unlockSkinTheme.m_0B1BA7C1 != 0)
133190
continue;
@@ -140,31 +197,17 @@ public void GenerateVoicelineMapping() {
140197

141198
var replacements = SkinTheme.GetReplacements(skinThemeGUID);
142199
foreach (var (_, newVoiceSetGuid) in replacements) {
143-
seenVoiceSets.Add(newVoiceSetGuid);
200+
SeenVoiceSets.TryAdd(newVoiceSetGuid, true);
144201
}
145202

146203
FindVoicelinesInVoiceSet(heroVoiceSetGuid, heroName, ref info, baseInfo, replacements);
147204
}
148205
}
149-
}
150-
151-
foreach (var guid in Program.TrackedFiles[0x5F]) {
152-
if (seenVoiceSets.Contains(guid)) {
153-
continue;
154-
}
155206

156-
var voiceSet = GetInstance<STUVoiceSet>(guid);
157-
if (voiceSet == null) continue;
158-
159-
var npcName = $"{IO.GetCleanString(voiceSet.m_269FC4E9)} {IO.GetCleanString(voiceSet.m_C0835C08)}".Trim();
160-
if (string.IsNullOrEmpty(npcName)) {
161-
npcName = IO.GetNullableGUIDName(guid) ?? $"Unknown{teResourceGUID.Index(guid):X}";
162-
}
163-
164-
Logger.Log($"Generating mapping for {npcName}");
165-
var info = new Combo.ComboInfo();
166-
FindVoicelinesInVoiceSet(guid, npcName, ref info);
167-
}
207+
heroTask.StopTask();
208+
task.Increment(1);
209+
ctx.Refresh();
210+
});
168211
}
169212

170213
private bool FindVoicelinesInVoiceSet(ulong? voiceSetGuid, string heroName, ref Combo.ComboInfo info, Combo.ComboInfo baseCombo = null, Dictionary<ulong, ulong> replacements = null) {

0 commit comments

Comments
 (0)