Skip to content

Commit 63a4be2

Browse files
PachiMakoPachiMako
committed
Revive --extract-shaders for dxbc bytecode extraction
teShaderCode, teShaderInstance and teShaderGroup all changed structure since at least the patch of Aug 10 '23 (Manifest 3320652641754470004). This change reran structural analysis of teShaderCode as well as teShaderGroup which now holds a list of shaders per group. teShaderInstance is not reversed in this change since --extract-shaders does not require it to be updated (note that teShaderInstance moved from 088 to 040). Further reversing of the fields and their associations may come in subsequent changes. Changes: - Improves error handling and logging for --extract-shaders - Adds errors in codepaths that are not longer functional such as resolving file names in SaveShaderGroup and resolving previously extracted shaders using their now stale hash - Fixup teSHADER_TYPE enum which is in fact a byte-sized bitflag with COMPUTE being inferred from no other stages being enabled - Minimal reverse of teShaderGroup and teShaderCode headers Verified working on overwatch build prometheus-2_23_0_0-150480 with valid dxbc for all entries of teShaderCode with the following distributions across shader types: PIXEL (2): 112210 VERTEX (1): 21777 COMPUTE (0): 147 GEOMETRY (4): 24 VERTEX, UNKNOWN_C (129): 13 GEOMETRY, UNKNOWN_B (20): 1
1 parent 3f2e672 commit 63a4be2

5 files changed

Lines changed: 135 additions & 102 deletions

File tree

‎DataTool/FindLogic/Combo.cs‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -671,19 +671,35 @@ public static ComboInfo Find(ComboInfo info, ulong guid, Dictionary<ulong, ulong
671671
materialInfo.m_materialIDs.Add(context.MaterialID);
672672
materialInfo.m_shaderSourceGUID = GetReplacement(material.Header.ShaderSource, replacements);
673673
materialInfo.m_shaderGroupGUID = GetReplacement(material.Header.ShaderGroup, replacements);
674-
try {
675-
if (Program.Flags.ExtractShaders) {
676-
teShaderGroup shaderGroup = new teShaderGroup(OpenFile(materialInfo.m_shaderGroupGUID));
677-
foreach (teResourceGUID shaderGuid in shaderGroup.Instances) {
678-
ulong shaderInstanceGuid = GetReplacement(shaderGuid, replacements);
679-
teShaderInstance shaderInstance = new teShaderInstance(OpenFile(shaderInstanceGuid));
680-
ulong shaderCodeGuid = GetReplacement(shaderInstance.Header.ShaderCode, replacements);
681-
teShaderCode shaderCode = new teShaderCode(OpenFile(shaderCodeGuid));
682-
materialInfo.m_shaders.Add((shaderInstanceGuid, shaderCodeGuid, shaderCode.ByteCode));
674+
675+
if (Program.Flags.ExtractShaders) {
676+
// Local function to prevent nesting on error handling, could be hoisted into a static function
677+
void ExtractShaders() {
678+
using Stream shaderGroupStream = OpenFile(materialInfo.m_shaderGroupGUID);
679+
if (shaderGroupStream == null) {
680+
Logger.Error("Combo", $"ExtractShaders: can't open shader group {materialInfo.m_shaderGroupGUID:X16}");
681+
return;
682+
}
683+
684+
teShaderGroup shaderGroup = new teShaderGroup(shaderGroupStream);
685+
if (shaderGroup.Shaders == null || shaderGroup.Instances == null) {
686+
Logger.Error("Combo", $"ExtractShaders: group {materialInfo.m_shaderGroupGUID:X16} has no shader/instance arrays");
687+
return;
688+
}
689+
690+
for (int i = 0; i < shaderGroup.Shaders.Length; i++) {
691+
ulong shaderCodeGuid = GetReplacement(shaderGroup.Shaders[i], replacements);
692+
using Stream shaderCodeStream = OpenFile(shaderCodeGuid);
693+
if (shaderCodeStream == null) {
694+
Logger.Error("Combo", $"ExtractShaders: can't open shader code {shaderCodeGuid:X16}");
695+
continue;
696+
}
697+
698+
teShaderCode shaderCode = new teShaderCode(shaderCodeStream);
699+
materialInfo.m_shaders.Add((GetReplacement(shaderGroup.Instances[i], replacements), shaderCodeGuid, shaderCode.ByteCode));
683700
}
684701
}
685-
} catch {
686-
// lol xd
702+
ExtractShaders();
687703
}
688704

689705
if (context.ModelLook == 0 && context.Model != 0) {

‎DataTool/ToolLogic/Extract/Debug/ExtractDebugShaders.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using TankLib.STU.Types;
1010
using static DataTool.Program;
1111
using static DataTool.Helper.STUHelper;
12+
using TankLib.Helpers;
1213

1314
namespace DataTool.ToolLogic.Extract.Debug;
1415

@@ -286,9 +287,10 @@ public static void SaveShaderGroup(teShaderGroup shaderGroup, string path) {
286287
int i = 0;
287288
foreach (ulong shaderGroupInstance in shaderGroup.Instances) {
288289
string name = null;
289-
if (shaderGroup.Hashes != null && shaderGroup.Hashes[i] != 0) {
290-
name = shaderGroup.Hashes[i].ToString("X8");
291-
}
290+
Logger.Error("SaveShaderGroup", $"teShaderGroup no longer associates a hash with teShaderInstance, see documentation on GetShaderByHash.");
291+
//if (shaderGroup.Hashes != null && shaderGroup.Hashes[i] != 0) {
292+
// name = shaderGroup.Hashes[i].ToString("X8");
293+
//}
292294

293295
SaveShaderInstance(path, shaderGroupInstance, name);
294296
i++;

‎TankLib/Enums.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@ public enum SDAM : long {
1212

1313
[Flags]
1414
[SuppressMessage("ReSharper", "InconsistentNaming")]
15-
public enum teSHADER_TYPE : ushort {
15+
public enum teSHADER_TYPE : byte {
16+
COMPUTE = 0,
1617
VERTEX = 1,
1718
PIXEL = 2,
1819
GEOMETRY = 4,
1920
UNKNOWN_A = 8,
2021
UNKNOWN_B = 16,
21-
UNKNOWN_C = 128,
22-
COMPUTE = 256
22+
UNKNOWN_C = 128
2323
}
2424

2525
[SuppressMessage("ReSharper", "InconsistentNaming")]

‎TankLib/teShaderCode.cs‎

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,32 +7,39 @@ namespace TankLib {
77
/// <summary>Tank ShaderCode, file type 087</summary>
88
public class teShaderCode {
99
/// <summary>ShaderCode Header </summary>
10-
[StructLayout(LayoutKind.Sequential, Pack = 4)]
11-
public struct ShaderCodeHeader {
12-
/// <summary>Offset to data</summary>
13-
public long DataOffset; // 0
14-
15-
/// <summary>Offset to unknown data</summary>
16-
public long OffsetB; // 8
17-
18-
public uint UnknownFF; // 16
19-
20-
/// <summary>Unknown data count</summary>
21-
public uint StreamOutDescCount; // 20
22-
23-
/// <summary>Size of compressed DXBC data</summary>
24-
public int CompressedSize; // 24
25-
26-
/// <summary>Size of decompressed DXBC data</summary>
27-
public int UncompressedSize; // 28
28-
29-
public uint Unknown; // 32
30-
31-
/// <summary>Shader type</summary>
32-
public Enums.teSHADER_TYPE ShaderType; // 36
33-
34-
// etc
35-
}
10+
[StructLayout(LayoutKind.Sequential, Pack = 1)]
11+
public unsafe struct ShaderCodeHeader {
12+
public ulong Const_0x00; // @0x00 (0) len 8 STATIC const = ffffffffffffffff
13+
public Enums.teSHADER_TYPE ShaderType; // @0x08 (8) len 1 - 0=compute, 1=vertex, 2=pixel, 4=geometry
14+
public byte Unk_0x09; // @0x09 (9) len 1 - variant/flags {0,1,4,5}, mostly 4
15+
public ushort Const_0x0A; // @0x0A (10) len 2 STATIC const = 0040
16+
public byte Enum_0x0C; // @0x0C (12) len 1 VAR(2) enum:2 e.g. 00
17+
public fixed byte Const_0x0D[3]; // @0x0D (13) len 3 STATIC const = 000000
18+
public byte Enum_0x10; // @0x10 (16) len 1 VAR(2) enum:2 e.g. 00
19+
public fixed byte Const_0x11[7]; // @0x11 (17) len 7 STATIC const = 00000000000000
20+
public byte Unk_0x18; // @0x18 (24) len 1 VAR(9) - e.g. 00
21+
public fixed byte Const_0x19[7]; // @0x19 (25) len 7 STATIC const = 00000000000000
22+
public ushort Offset_0x20; // @0x20 (32) len 2 VAR(53) offset?:LE e.g. 0000
23+
public fixed byte Const_0x22[6]; // @0x22 (34) len 6 STATIC const = 000000000000
24+
public fixed byte Unk_0x28[12]; // @0x28 (40) len 12 VAR(4906) - e.g. 000000000000000000000000
25+
public ushort Const_0x34; // @0x34 (52) len 2 STATIC const = 0000
26+
public ushort Unk_0x36; // @0x36 (54) len 2 VAR(1038) - e.g. 0000
27+
public fixed byte Const_0x38[16]; // @0x38 (56) len 16 STATIC const = 00000000000000000000000000000000
28+
public ushort Offset_0x48; // @0x48 (72) len 2 VAR(34) offset?:LE e.g. 0000
29+
public fixed byte Const_0x4A[6]; // @0x4A (74) len 6 STATIC const = 000000000000
30+
public ulong Unk_0x50; // @0x50 (80) len 8 VAR(202) - e.g. 0000000000000000
31+
public ulong Const_0x58; // @0x58 (88) len 8 STATIC const = 0000000000000000
32+
public ulong Hash_0x60; // @0x60 (96) len 8 VAR(123805) hash/opaque e.g. 00001061e477405c
33+
public ulong Const_0x68; // @0x68 (104) len 8 STATIC const = 0000000000000000
34+
public ushort DataOffset; // @0x70 (112) len 2 VAR(145) offset?:LE e.g. 0001
35+
public fixed byte Unk_0x72[18]; // @0x72 (114) len 18 VAR(2) - e.g. 000000000000000000000000000000000000
36+
public ushort CompressedSize; // @0x84 (132) len 2 VAR(9510) ~size:r+1.00:LE e.g. 0001
37+
public ushort Const_0x86; // @0x86 (134) len 2 STATIC const = 0000
38+
public ushort UncompressedSize; // @0x88 (136) len 2 VAR(10057) ~size:r+0.99:LE e.g. 0002
39+
public uint Enum_0x8A; // @0x8A (138) len 4 VAR(3) enum:3 e.g. 00000000
40+
public byte Unk_0x8E; // @0x8E (142) len 1 VAR(15) - e.g. 00
41+
public fixed byte Const_0x8F[13]; // @0x8F (143) len 13 STATIC const = ffffffffff0000000000000000
42+
} // size = 156 (0x9C)
3643

3744
/// <summary>Header Data</summary>
3845
public ShaderCodeHeader Header;

‎TankLib/teShaderGroup.cs‎

Lines changed: 67 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,63 @@
11
using System.IO;
22
using System.Runtime.InteropServices;
3+
using TankLib.Helpers;
34

45
namespace TankLib {
56
/// <summary>Tank ShaderGroup, file type 085</summary>
67
public class teShaderGroup {
7-
[StructLayout(LayoutKind.Sequential, Pack = 4)]
8-
public struct GroupHeader {
9-
/// <summary>ShaderInstance array offset</summary>
10-
public long InstanceOffset; // 0
11-
12-
public long HashOffset; // 8
13-
public long FlagsOffset; // 16
14-
15-
public long NewIdk; // n/a -> 24
16-
17-
public long OffsetD; // 24 -> 32
18-
public long OffsetE; // 32 -> 40
19-
20-
/// <summary>teShaderSource that this group was generated from</summary>
21-
/// <remarks>088 GUID</remarks>
22-
public teResourceGUID SourceGUID; // 40 -> 48
23-
24-
/// <summary>A virtual reference. Usage unknown</summary>
25-
/// <remarks>00F GUID</remarks>
26-
public teResourceGUID CacheGUID; // 48 -> 56
27-
28-
public ulong Flags; // 56 -> 64
29-
30-
public int NumIdk; // n/a -> 72
31-
public int NumShaders; // 64 -> 76
32-
33-
// ...
34-
}
8+
[StructLayout(LayoutKind.Sequential, Pack = 1)]
9+
public unsafe struct GroupHeader {
10+
public ushort Enum_0x00; // @0x00 (0) len 2 VAR(8) enum:8 e.g. bdef
11+
public byte Const_0x02; // @0x02 (2) len 1 STATIC const = ff
12+
public byte Enum_0x03; // @0x03 (3) len 1 VAR(4) enum:4 e.g. 6b
13+
public ushort Const_0x04; // @0x04 (4) len 2 STATIC const = fcff
14+
public byte Enum_0x06; // @0x06 (6) len 1 VAR(2) enum:2 e.g. c0
15+
public fixed byte Const_0x07[9]; // @0x07 (7) len 9 STATIC const = ffb800000000000000
16+
public ushort Size_0x10; // @0x10 (16) len 2 VAR(37) ~size:r+1.00:LE e.g. 0001
17+
public fixed byte Const_0x12[16]; // @0x12 (18) len 16 STATIC const = 000000000000ffffffffffffffff0000
18+
public uint Unk_0x22; // @0x22 (34) len 4 VAR(20) - e.g. 0100ffff
19+
public ushort Const_0x26; // @0x26 (38) len 2 STATIC const = ffff
20+
public byte Enum_0x28; // @0x28 (40) len 1 VAR(2) enum:2 e.g. 00
21+
public byte Const_0x29; // @0x29 (41) len 1 STATIC const = 00
22+
public uint Unk_0x2A; // @0x2A (42) len 4 VAR(20) - e.g. 0100ffff
23+
public ushort Const_0x2E; // @0x2E (46) len 2 STATIC const = ffff
24+
public byte Mirror_0x30; // @0x30 (48) len 1 VAR(2) =field@40, enum:2 e.g. 00
25+
public byte Const_0x31; // @0x31 (49) len 1 STATIC const = 00
26+
public uint Mirror_0x32; // @0x32 (50) len 4 VAR(20) =field@42 e.g. 0100ffff
27+
public fixed byte Const_0x36[10]; // @0x36 (54) len 10 STATIC const = ffff0000000000000000
28+
public byte Unk_0x40; // @0x40 (64) len 1 VAR(18) - e.g. 01
29+
public byte Const_0x41; // @0x41 (65) len 1 STATIC const = 00
30+
public byte Unk_0x42; // @0x42 (66) len 1 VAR(21) - e.g. 00
31+
public byte Const_0x43; // @0x43 (67) len 1 STATIC const = 00
32+
public byte Enum_0x44; // @0x44 (68) len 1 VAR(2) enum:2 e.g. 00
33+
public fixed byte Const_0x45[3]; // @0x45 (69) len 3 STATIC const = 000000
34+
public byte Mirror_0x48; // @0x48 (72) len 1 VAR(18) =field@64 e.g. 01
35+
public byte Const_0x49; // @0x49 (73) len 1 STATIC const = 00
36+
public byte Unk_0x4A; // @0x4A (74) len 1 VAR(20) - e.g. 00
37+
public byte Const_0x4B; // @0x4B (75) len 1 STATIC const = 00
38+
public byte Mirror_0x4C; // @0x4C (76) len 1 VAR(2) =field@68, enum:2 e.g. 00
39+
public fixed byte Const_0x4D[3]; // @0x4D (77) len 3 STATIC const = 000000
40+
public byte Mirror_0x50; // @0x50 (80) len 1 VAR(18) =field@64 e.g. 01
41+
public byte Const_0x51; // @0x51 (81) len 1 STATIC const = 00
42+
public byte Mirror_0x52; // @0x52 (82) len 1 VAR(20) =field@74 e.g. 00
43+
public byte Const_0x53; // @0x53 (83) len 1 STATIC const = 00
44+
public byte Mirror_0x54; // @0x54 (84) len 1 VAR(2) =field@68, enum:2 e.g. 00
45+
public fixed byte Const_0x55[3]; // @0x55 (85) len 3 STATIC const = 000000
46+
public ushort Size_0x58; // @0x58 (88) len 2 VAR(37) ~size:r+1.00:LE e.g. 1801
47+
public fixed byte Const_0x5A[6]; // @0x5A (90) len 6 STATIC const = 000000000000
48+
public int Unk_0x60; // @0x60 (96) len 4
49+
public int NumShaders; // @0x64 (100) len 4
50+
public ushort ShadersOffset; // @0x68 (104) len 2 VAR(37) ~size:r+1.00:LE e.g. 0001
51+
public fixed byte Const_0x6A[6]; // @0x6A (106) len 6 STATIC const = 000000000000
52+
public ushort Size_0x70; // @0x70 (112) len 2 VAR(37) ~size:r+1.00:LE e.g. 000a
53+
public fixed byte Const_0x72[6]; // @0x72 (114) len 6 STATIC const = 000000000000
54+
public ushort InstancesOffset; // @0x78 (120) len 2 VAR(37) ~size:r+1.00:LE e.g. 0005
55+
public fixed byte Const_0x7A[6]; // @0x7A (122) len 6 STATIC const = 000000000000
56+
public ushort Size_0x80; // @0x80 (128) len 2 VAR(37) ~size:r+1.00:LE e.g. 0004
57+
public fixed byte Const_0x82[22]; // @0x82 (130) len 22 STATIC const = 00000000000000000000000000000000000000000000
58+
public ushort Offset_0x98; // @0x98 (152) len 2 VAR(43) offset?:LE e.g. 0d01
59+
public fixed byte Const_0x9A[30]; // @0x9A (154) len 30 STATIC const = 00000000100e00000000000000000000000000000000ffffffffff000000
60+
} // size = 184 (0xB8)
3561

3662
[StructLayout(LayoutKind.Sequential, Pack = 4)]
3763
public struct ShaderQuality {
@@ -54,17 +80,15 @@ public struct ShaderUnk {
5480
/// <summary>Header Data</summary>
5581
public GroupHeader Header;
5682

57-
/// <summary>ShaderInstances</summary>
83+
/// <summary>teShaderCode GUIDs (087)</summary>
84+
public teResourceGUID[] Shaders;
85+
86+
/// <summary>teShaderInstance info GUIDs (040)</summary>
5887
public teResourceGUID[] Instances;
5988

6089
/// <summary>Flags that are used to select the correct ShaderInstance pair based on parameters</summary>
6190
public ulong[] InstanceFlags;
6291

63-
/// <summary>
64-
/// ShaderInstance "hashes". Used to identify instances in 088 groups.
65-
/// </summary>
66-
public uint[] Hashes;
67-
6892
public ShaderQuality[] ShaderQualities;
6993
public ShaderUnk[] ShaderUnks;
7094

@@ -89,43 +113,27 @@ public teShaderGroup(BinaryReader reader) {
89113
private void Read(BinaryReader reader) {
90114
Header = reader.Read<GroupHeader>();
91115

92-
if (Header.InstanceOffset > 0) {
93-
reader.BaseStream.Position = Header.InstanceOffset;
94-
Instances = reader.ReadArray<teResourceGUID>(Header.NumShaders);
95-
}
96-
97-
if (Header.HashOffset > 0) {
98-
reader.BaseStream.Position = Header.HashOffset;
99-
Hashes = reader.ReadArray<uint>(Header.NumShaders);
116+
if (Header.ShadersOffset > 0) {
117+
reader.BaseStream.Position = Header.ShadersOffset;
118+
Shaders = reader.ReadArray<teResourceGUID>(Header.NumShaders);
100119
}
101120

102-
if (Header.FlagsOffset > 0) {
103-
reader.BaseStream.Position = Header.FlagsOffset;
104-
InstanceFlags = reader.ReadArray<ulong>(Header.NumShaders);
105-
}
106-
107-
{
108-
reader.BaseStream.Position = 72;
109-
ShaderQualities = reader.ReadArray<ShaderQuality>(5);
110-
111-
reader.BaseStream.Position = 104;
112-
ShaderUnks = reader.ReadArray<ShaderUnk>(5);
121+
if (Header.InstancesOffset > 0) {
122+
reader.BaseStream.Position = Header.InstancesOffset;
123+
Instances = reader.ReadArray<teResourceGUID>(Header.NumShaders);
113124
}
114125
}
115126

116127
/// <summary>
128+
/// DEPRECATED: teShaderGroup used to store a hash to match with a specific teShaderInstance. This mapping has since been broken as
129+
/// teShaderGroup now holds guids to both teShaderInstance and teShaderCode mapping them with a simple positional index instead.
117130
/// Get a ShaderInstance GUID from a "hash"
118131
/// </summary>
119132
/// <note>Mostly used on 088 groups</note>
120133
/// <param name="hash">"Hash" associated with a specific ShaderInstance</param>
121134
/// <returns></returns>
122135
public teResourceGUID GetShaderByHash(uint hash) {
123-
if (Hashes == null) return (teResourceGUID) 0;
124-
for (int i = 0; i < Header.NumShaders; i++) {
125-
if (Hashes[i] == hash) {
126-
return Instances[i];
127-
}
128-
}
136+
Logger.Error("teShaderGroup", $"GetShaderByHash doesn't work any longer for {hash:X16}, see documentation on GetShaderByHash.");
129137
return (teResourceGUID) 0;
130138
}
131139
}

0 commit comments

Comments
 (0)