Skip to content

Commit 999fa2a

Browse files
blueskythlikescloudsblueskythlikesclouds
authored
Implement shader archive decompressor for the build system. (#466)
* Implement shader archive decompressor for the build system. * Fix Linux compilation error.
1 parent 821f5eb commit 999fa2a

5 files changed

Lines changed: 274 additions & 1 deletion

File tree

‎UnleashedRecompLib/CMakeLists.txt‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,15 @@ add_custom_command(
3737
"${CMAKE_CURRENT_SOURCE_DIR}/config/SWA.toml"
3838
)
3939
40+
add_custom_command(
41+
OUTPUT
42+
"${CMAKE_CURRENT_SOURCE_DIR}/private/shader_decompressed.ar"
43+
COMMAND
44+
$<TARGET_FILE:x_decompress> "${CMAKE_CURRENT_SOURCE_DIR}/private/shader.ar" "${CMAKE_CURRENT_SOURCE_DIR}/private/shader_decompressed.ar"
45+
DEPENDS
46+
"${CMAKE_CURRENT_SOURCE_DIR}/private/shader.ar"
47+
)
48+
4049
set(XENOS_RECOMP_ROOT "${UNLEASHED_RECOMP_TOOLS_ROOT}/XenosRecomp/XenosRecomp")
4150
set(XENOS_RECOMP_INCLUDE "${XENOS_RECOMP_ROOT}/shader_common.h")
4251
@@ -58,7 +67,7 @@ add_custom_command(
5867
$<TARGET_FILE:XenosRecomp>
5968
DEPENDS
6069
"${CMAKE_CURRENT_SOURCE_DIR}/private/default_patched.xex"
61-
"${CMAKE_CURRENT_SOURCE_DIR}/private/shader.ar"
70+
"${CMAKE_CURRENT_SOURCE_DIR}/private/shader_decompressed.ar"
6271
${XENOS_RECOMP_SOURCES}
6372
${XENOS_RECOMP_INCLUDE}
6473
)

‎flatpak/io.github.hedge_dev.unleashedrecomp.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@
5252
"type": "file",
5353
"path": "private/shader.ar",
5454
"dest": "UnleashedRecompLib/private"
55+
},
56+
{
57+
"type": "file",
58+
"path": "private/shader_decompressed.ar",
59+
"dest": "UnleashedRecompLib/private"
5560
}
5661
],
5762
"build-options": {

‎tools/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/bc_diff)
22
add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/file_to_c)
33
add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/fshasher)
4+
add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/x_decompress)
45
add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/XenonRecomp)
56
add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/XenosRecomp)

‎tools/x_decompress/CMakeLists.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
project("x_decompress")
2+
set(CMAKE_CXX_STANDARD 17)
3+
4+
add_executable(x_decompress
5+
"x_decompress.cpp"
6+
"${UNLEASHED_RECOMP_TOOLS_ROOT}/XenonRecomp/thirdparty/libmspack/libmspack/mspack/lzxd.c"
7+
)
8+
9+
target_include_directories(x_decompress
10+
PRIVATE "${UNLEASHED_RECOMP_TOOLS_ROOT}/XenonRecomp/thirdparty/libmspack/libmspack/mspack"
11+
)

‎tools/x_decompress/x_decompress.cpp‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
#include <algorithm>
2+
#include <cstdint>
3+
#include <cstddef>
4+
#include <cstring>
5+
#include <vector>
6+
#include <fstream>
7+
#include <cassert>
8+
#include <mspack.h>
9+
#include <lzx.h>
10+
11+
static std::vector<uint8_t> readAllBytes(const char* path)
12+
{
13+
std::ifstream file{ path, std::ios::binary };
14+
std::vector<uint8_t> result{};
15+
16+
if (!file.good()) {
17+
return result;
18+
}
19+
20+
file.seekg(0, std::ios::end);
21+
result.resize(file.tellg());
22+
23+
file.seekg(0, std::ios::beg);
24+
file.read(reinterpret_cast<char*>(result.data()), result.size());
25+
26+
return result;
27+
}
28+
29+
template<typename T>
30+
static T byteSwap(T value)
31+
{
32+
if constexpr (sizeof(T) == 1)
33+
return value;
34+
else if constexpr (sizeof(T) == 2)
35+
return static_cast<T>(__builtin_bswap16(static_cast<uint16_t>(value)));
36+
else if constexpr (sizeof(T) == 4)
37+
return static_cast<T>(__builtin_bswap32(static_cast<uint32_t>(value)));
38+
else if constexpr (sizeof(T) == 8)
39+
return static_cast<T>(__builtin_bswap64(static_cast<uint64_t>(value)));
40+
41+
assert(false && "Unexpected byte size.");
42+
return value;
43+
}
44+
45+
template<typename T>
46+
static void byteSwapInplace(T& value)
47+
{
48+
value = byteSwap(value);
49+
}
50+
51+
struct ReadStream
52+
{
53+
const uint8_t* data = nullptr;
54+
int size = 0; // Size from every compressed block.
55+
};
56+
57+
static int mspackRead(mspack_file* file, void* buffer, int bytes)
58+
{
59+
ReadStream* stream = reinterpret_cast<ReadStream*>(file);
60+
61+
if (stream->size == 0)
62+
{
63+
uint16_t size = byteSwap(*reinterpret_cast<const uint16_t*>(stream->data));
64+
stream->data += sizeof(uint16_t);
65+
66+
// This indicates there is an uncompressed block size available. We don't need it so we skip it.
67+
if ((size & 0xFF00) == 0xFF00)
68+
{
69+
stream->data += 1;
70+
size = byteSwap(*reinterpret_cast<const uint16_t*>(stream->data));
71+
stream->data += sizeof(uint16_t);
72+
}
73+
74+
stream->size = size;
75+
}
76+
77+
int sizeToRead = std::min(stream->size, bytes);
78+
79+
memcpy(buffer, stream->data, sizeToRead);
80+
stream->data += sizeToRead;
81+
stream->size -= sizeToRead;
82+
83+
return sizeToRead;
84+
}
85+
86+
struct WriteStream
87+
{
88+
uint8_t* data = nullptr;
89+
std::size_t size = 0; // Remaining available space in the stream.
90+
};
91+
92+
static int mspackWrite(mspack_file* file, void* buffer, int bytes)
93+
{
94+
WriteStream* stream = reinterpret_cast<WriteStream*>(file);
95+
96+
std::size_t sizeToWrite = std::min(stream->size, static_cast<std::size_t>(bytes));
97+
98+
memcpy(stream->data, buffer, sizeToWrite);
99+
stream->data += sizeToWrite;
100+
stream->size -= sizeToWrite;
101+
102+
return static_cast<int>(sizeToWrite);
103+
}
104+
105+
static void* mspackAlloc(mspack_system* self, size_t bytes)
106+
{
107+
return operator new(bytes);
108+
}
109+
110+
static void mspackFree(void* ptr)
111+
{
112+
operator delete(ptr);
113+
}
114+
115+
static void mspackCopy(void* src, void* dst, size_t bytes)
116+
{
117+
memcpy(dst, src, bytes);
118+
}
119+
120+
static mspack_system g_lzxSystem =
121+
{
122+
nullptr,
123+
nullptr,
124+
mspackRead,
125+
mspackWrite,
126+
nullptr,
127+
nullptr,
128+
nullptr,
129+
mspackAlloc,
130+
mspackFree,
131+
mspackCopy
132+
};
133+
134+
// Xbox Compression header definitions.
135+
static constexpr uint32_t XCompressSignature = 0xFF512EE;
136+
137+
struct XCompressHeader
138+
{
139+
uint32_t signature;
140+
uint32_t field04;
141+
uint32_t field08;
142+
uint32_t field0C;
143+
uint32_t windowSize;
144+
uint32_t compressedBlockSize;
145+
uint64_t uncompressedSize;
146+
uint64_t compressedSize;
147+
uint32_t uncompressedBlockSize;
148+
uint32_t field2C;
149+
150+
void byteSwap()
151+
{
152+
byteSwapInplace(signature);
153+
byteSwapInplace(field04);
154+
byteSwapInplace(field08);
155+
byteSwapInplace(field0C);
156+
byteSwapInplace(windowSize);
157+
byteSwapInplace(compressedBlockSize);
158+
byteSwapInplace(uncompressedSize);
159+
byteSwapInplace(compressedSize);
160+
byteSwapInplace(uncompressedBlockSize);
161+
byteSwapInplace(field2C);
162+
}
163+
};
164+
165+
int main(int argc, char** argv)
166+
{
167+
if (argc < 3)
168+
{
169+
printf("Usage: x_decompress [input file path] [output file path]");
170+
return EXIT_SUCCESS;
171+
}
172+
173+
std::vector<uint8_t> file = readAllBytes(argv[1]);
174+
if (file.empty())
175+
{
176+
fprintf(stderr, "Input file \"%s\" not found or empty", argv[1]);
177+
return EXIT_FAILURE;
178+
}
179+
180+
std::vector<uint8_t> decompressedFile;
181+
182+
if (file.size() >= sizeof(XCompressHeader) && byteSwap(*reinterpret_cast<uint32_t*>(file.data())) == XCompressSignature)
183+
{
184+
XCompressHeader* header = reinterpret_cast<XCompressHeader*>(file.data());
185+
header->byteSwap();
186+
187+
decompressedFile.resize(header->uncompressedSize);
188+
189+
const uint8_t* srcBytes = file.data() + sizeof(XCompressHeader);
190+
191+
WriteStream dstStream;
192+
dstStream.data = decompressedFile.data();
193+
dstStream.size = decompressedFile.size();
194+
195+
// libmspack wants the bit index. This value is always guaranteed to be a power of two,
196+
// so we can extract the bit index by counting the amount of leading zeroes.
197+
int windowBits = 0;
198+
uint32_t windowSize = header->windowSize;
199+
while ((windowSize & 0x1) == 0)
200+
{
201+
++windowBits;
202+
windowSize >>= 1;
203+
}
204+
205+
// Loop over compressed blocks.
206+
while (srcBytes < (file.data() + file.size()) && dstStream.data < (decompressedFile.data() + decompressedFile.size()))
207+
{
208+
uint32_t compressedSize = byteSwap(*reinterpret_cast<const uint32_t*>(srcBytes));
209+
srcBytes += sizeof(uint32_t);
210+
211+
ReadStream srcStream;
212+
srcStream.data = srcBytes;
213+
214+
std::size_t uncompressedBlockSize = std::min(static_cast<std::size_t>(header->uncompressedBlockSize), dstStream.size);
215+
216+
lzxd_stream* lzx = lzxd_init(
217+
&g_lzxSystem,
218+
reinterpret_cast<mspack_file*>(&srcStream),
219+
reinterpret_cast<mspack_file*>(&dstStream),
220+
windowBits,
221+
0,
222+
static_cast<int>(header->compressedBlockSize),
223+
static_cast<off_t>(uncompressedBlockSize),
224+
0);
225+
226+
lzxd_decompress(lzx, uncompressedBlockSize);
227+
lzxd_free(lzx);
228+
229+
srcBytes += compressedSize;
230+
}
231+
}
232+
else
233+
{
234+
decompressedFile = std::move(file);
235+
}
236+
237+
std::ofstream outputFile(argv[2], std::ios::binary);
238+
if (!outputFile.good())
239+
{
240+
fprintf(stderr, "Cannot open output file \"%s\" for writing", argv[2]);
241+
return EXIT_FAILURE;
242+
}
243+
244+
outputFile.write(reinterpret_cast<const char*>(decompressedFile.data()), decompressedFile.size());
245+
246+
return EXIT_SUCCESS;
247+
}

0 commit comments

Comments
 (0)