read assets from ucas file

Author: moritz-h

libsave/include/SatisfactorySave/GameTypes/UE/Core/IO/IoStore.h

@@ -65,6 +65,32 @@ namespace SatisfactorySave {
     struct SATISFACTORYSAVE_API FIoStoreTocCompressedBlockEntry {
         std::array<uint8_t, 5 + 3 + 3 + 1> Data{};
 
+        static constexpr uint32_t OffsetBits = 40;
+        static constexpr uint64_t OffsetMask = (1ull << OffsetBits) - 1ull;
+        static constexpr uint32_t SizeBits = 24;
+        static constexpr uint32_t SizeMask = (1 << SizeBits) - 1;
+        static constexpr uint32_t SizeShift = 8;
+
+        [[nodiscard]] inline uint64_t GetOffset() const {
+            const auto* Offset = reinterpret_cast<const uint64_t*>(Data.data());
+            return *Offset & OffsetMask;
+        }
+
+        [[nodiscard]] inline uint32_t GetCompressedSize() const {
+            const auto* Size = reinterpret_cast<const uint32_t*>(Data.data()) + 1;
+            return (*Size >> SizeShift) & SizeMask;
+        }
+
+        [[nodiscard]] inline uint32_t GetUncompressedSize() const {
+            const auto* UncompressedSize = reinterpret_cast<const uint32_t*>(Data.data()) + 2;
+            return *UncompressedSize & SizeMask;
+        }
+
+        [[nodiscard]] inline uint8_t GetCompressionMethodIndex() const {
+            const auto* Index = reinterpret_cast<const uint32_t*>(Data.data()) + 2;
+            return static_cast<uint8_t>(*Index >> SizeBits);
+        }
+
         void serialize(Archive& ar) {
             ar << Data;
         }

libsave/include/SatisfactorySave/Pak/IoStoreFile.h

@@ -47,5 +47,6 @@ namespace SatisfactorySave {
     private:
         FIoStoreTocResource utoc_;
         std::unique_ptr<DirectoryIndexReader> dirIndex_;
+        std::unique_ptr<IFStreamArchive> ucasAr_;
     };
 } // namespace SatisfactorySave

libsave/src/IO/OodleUtils.cpp

@@ -0,0 +1,48 @@
+#include "OodleUtils.h"
+
+#include <cstdint>
+#include <stdexcept>
+
+#ifdef _WIN32
+#include <Windows.h>
+#endif
+
+std::vector<char> SatisfactorySave::oodleDecompress(const std::vector<char>& buffer, std::size_t decompressed_size) {
+    std::vector<char> buffer_uncompressed(decompressed_size);
+    oodleDecompress(buffer_uncompressed.data(), decompressed_size, buffer.data(), buffer.size());
+    return buffer_uncompressed;
+}
+
+#ifdef _WIN32
+
+typedef intptr_t(__stdcall* OodleLZ_DecompressFunc)(const void* compBuf, intptr_t compBufSize, void* rawBuf,
+    intptr_t rawLen, int32_t fuzzSafe, int32_t checkCRC, int32_t verbosity, void* decBufBase, intptr_t decBufSize,
+    void* fpCallback, void* callbackUserData, void* decoderMemory, intptr_t decoderMemorySize, int32_t threadPhase);
+
+void SatisfactorySave::oodleDecompress(char* dest, std::size_t destLen, const char* source, std::size_t sourceLen) {
+    static OodleLZ_DecompressFunc OodleLZ_Decompress = nullptr;
+    if (OodleLZ_Decompress == nullptr) {
+        HMODULE handle = LoadLibrary(TEXT("oo2core_9_win64.dll"));
+        if (handle == NULL) {
+            throw std::runtime_error("Error loading Oodle DLL!");
+        }
+        OodleLZ_Decompress = (OodleLZ_DecompressFunc) GetProcAddress(handle, "OodleLZ_Decompress");
+        if (OodleLZ_Decompress == NULL) {
+            throw std::runtime_error("Error loading OodleLZ_Decompress!");
+        }
+    }
+
+    auto len = OodleLZ_Decompress(source, static_cast<int64_t>(sourceLen), dest, static_cast<int64_t>(destLen), 1, 0, 0,
+        nullptr, 0, nullptr, nullptr, nullptr, 0, 3);
+    if (len != destLen) {
+        throw std::runtime_error("Error decompressing Oodle!");
+    }
+}
+
+#else
+
+void SatisfactorySave::oodleDecompress(char* dest, std::size_t destLen, const char* source, std::size_t sourceLen) {
+    throw std::runtime_error("Not implemented!");
+}
+
+#endif

libsave/src/IO/OodleUtils.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <vector>
+
+namespace SatisfactorySave {
+    std::vector<char> oodleDecompress(const std::vector<char>& buffer, std::size_t decompressed_size);
+
+    void oodleDecompress(char* dest, std::size_t destLen, const char* source, std::size_t sourceLen);
+} // namespace SatisfactorySave

libsave/src/Pak/IoStoreFile.cpp

@@ -4,6 +4,7 @@
 
 #include "GameTypes/UE/Core/IO/IoStore.h"
 #include "IO/MemoryStreams.h"
+#include "IO/OodleUtils.h"
 
 SatisfactorySave::DirectoryIndexReader::DirectoryIndexReader(const FIoDirectoryIndexResource& res) : res_(res) {
     if (!res_.MountPoint.starts_with("../../../")) {
@@ -42,7 +43,12 @@ void SatisfactorySave::DirectoryIndexReader::parseFile(uint32_t file_idx, const
 
 SatisfactorySave::IoStoreFile::IoStoreFile(const std::filesystem::path& path) {
     if (!std::filesystem::is_regular_file(path)) {
-        throw std::runtime_error("IoStore file invalid: " + path.string());
+        throw std::runtime_error("IoStore utoc file invalid: " + path.string());
+    }
+    std::filesystem::path ucas_path = path;
+    ucas_path.replace_extension(std::filesystem::path(".ucas"));
+    if (!std::filesystem::is_regular_file(ucas_path)) {
+        throw std::runtime_error("IoStore ucas file invalid: " + ucas_path.string());
     }
 
     IFStreamArchive utocAr(path);
@@ -64,6 +70,8 @@ SatisfactorySave::IoStoreFile::IoStoreFile(const std::filesystem::path& path) {
 
         dirIndex_ = std::make_unique<DirectoryIndexReader>(dirIndexRes);
     }
+
+    ucasAr_ = std::make_unique<IFStreamArchive>(ucas_path);
 }
 
 std::vector<std::string> SatisfactorySave::IoStoreFile::getAllAssetFilenames() const {
@@ -80,6 +88,52 @@ std::vector<std::string> SatisfactorySave::IoStoreFile::getAllAssetFilenames() c
 }
 
 std::vector<char> SatisfactorySave::IoStoreFile::readAssetFileContent(const std::string& filename) {
-    // TODO
-    return {};
+    if (dirIndex_ == nullptr) {
+        throw std::runtime_error("Missing directory index!");
+    }
+    if (!dirIndex_->directoryEntries().contains(filename)) {
+        throw std::runtime_error("Asset file not found in utoc: " + filename);
+    }
+
+    // UserData of directory index file entry is index to chunk data
+    uint32_t chunkIdx = dirIndex_->directoryEntries().at(filename);
+    const auto& chunk = utoc_.ChunkOffsetLengths[chunkIdx];
+
+    // The idea behind chunk data seems to be that all data is stored in a single address space. The address space is
+    // divided into blocks. All data seems aligned to a block start. Each block is then compressed individually. The
+    // ucas stores the compressed blocks directly after each other. Divide address space offset and length of chunk
+    // data by the block size to get index and number of compression blocks.
+    const auto& blockSize = utoc_.Header.CompressionBlockSize;
+    const uint64_t compBlockIdx = chunk.GetOffset() / blockSize;
+    const uint32_t compBlockNum = (chunk.GetLength() + blockSize - 1) / blockSize;
+
+    // Allocate output
+    std::vector<char> buf(chunk.GetLength());
+
+    auto& ar = *ucasAr_;
+
+    // Loop over compression blocks
+    for (uint32_t i = 0; i < compBlockNum; i++) {
+        const auto& compBlock = utoc_.CompressionBlocks[compBlockIdx + i];
+
+        // Validate that uncompressed size of all compression blocks (except the last one) is block size.
+        if (i < compBlockNum - 1 && compBlock.GetUncompressedSize() != blockSize) {
+            throw std::runtime_error("Unexpected uncompressed block size");
+        }
+
+        ar.seek(compBlock.GetOffset());
+        const auto compMethod = utoc_.CompressionMethods.at(compBlock.GetCompressionMethodIndex());
+        if (compMethod == "None") {
+            // Uncompressed, copy directoy to out buffer.
+            ar.serializeRaw(buf.data() + i * blockSize, compBlock.GetCompressedSize());
+        } else if (compMethod == "Oodle") {
+            const auto comp_buf = ar.read_buffer(compBlock.GetCompressedSize());
+            oodleDecompress(buf.data() + i * blockSize, compBlock.GetUncompressedSize(), comp_buf.data(),
+                compBlock.GetCompressedSize());
+        } else {
+            throw std::runtime_error("Unknown compression method: " + compMethod);
+        }
+    }
+
+    return buf;
 }