Skip to content

Commit abca13b

Browse files
moritz-hmoritz-h
committed
decompress asset content on demand
1 parent 076e972 commit abca13b

7 files changed

Lines changed: 145 additions & 41 deletions

File tree

‎libsave/include/SatisfactorySave/IO/Archive/IStreamArchive.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ namespace SatisfactorySave {
1313

1414
class SATISFACTORYSAVE_API IStreamArchive : public Archive {
1515
public:
16+
explicit IStreamArchive(std::unique_ptr<IStream> istream) : istream_(std::move(istream)) {}
17+
1618
explicit IStreamArchive(std::vector<char>&& buf) {
1719
istream_ = std::make_unique<MemoryBufferIStream>(std::move(buf));
1820
}

‎libsave/include/SatisfactorySave/Pak/AbstractPakFile.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
#pragma once
22

3+
#include <memory>
34
#include <string>
45
#include <vector>
56

7+
#include "../IO/IOStream.h"
68
#include "AssetFile.h"
79
#include "satisfactorysave_export.h"
810

911
namespace SatisfactorySave {
1012
class PakManager;
1113

12-
class SATISFACTORYSAVE_API AbstractPakFile {
14+
class SATISFACTORYSAVE_API AbstractPakFile : public std::enable_shared_from_this<AbstractPakFile> {
1315
public:
1416
explicit AbstractPakFile(std::shared_ptr<PakManager> pakManager) : pakManager_(std::move(pakManager)) {}
1517
virtual ~AbstractPakFile() = default;
@@ -20,6 +22,8 @@ namespace SatisfactorySave {
2022

2123
virtual std::vector<char> readAssetFileContent(const std::string& filename) = 0;
2224

25+
virtual std::unique_ptr<IStream> getAssetFileStream(const std::string& filename);
26+
2327
AssetFile readAsset(const std::string& filename);
2428

2529
protected:

‎libsave/include/SatisfactorySave/Pak/AssetFile.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "../GameTypes/UE/CoreUObject/Serialization/AsyncLoading2.h"
1111
#include "../GameTypes/UE/CoreUObject/Serialization/ZenPackageHeader.h"
1212
#include "../IO/Archive/IStreamArchive.h"
13+
#include "../IO/IOStream.h"
1314
#include "AssetExport.h"
1415
#include "satisfactorysave_export.h"
1516

@@ -18,8 +19,8 @@ namespace SatisfactorySave {
1819

1920
class SATISFACTORYSAVE_API AssetFile : public IStreamArchive {
2021
public:
21-
AssetFile(std::shared_ptr<PakManager> pakManager, std::vector<char>&& uassetData,
22-
std::vector<char>&& ubulkData = {});
22+
AssetFile(std::shared_ptr<PakManager> pakManager, std::unique_ptr<IStream> uassetStream,
23+
std::unique_ptr<IStream> ubulkStream = nullptr);
2324

2425
inline void seekCookedSerialOffset(uint64_t offset) {
2526
seek(packageHeader_.PackageSummary.HeaderSize + offset);

‎libsave/include/SatisfactorySave/Pak/IoStoreFile.h‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,89 @@ namespace SatisfactorySave {
4444

4545
std::vector<char> readAssetFileContent(const std::string& filename) override;
4646

47+
std::unique_ptr<IStream> getAssetFileStream(const std::string& filename) override;
48+
4749
std::vector<char> readChunkContent(std::size_t chunkIdx);
4850

4951
private:
52+
class SATISFACTORYSAVE_API AssetFileIStream : public IStream {
53+
public:
54+
explicit AssetFileIStream(std::shared_ptr<IoStoreFile> ioStore, uint32_t chunkIdx)
55+
: ioStore_(std::move(ioStore)),
56+
pos_(0) {
57+
const auto& chunk = ioStore_->utoc_.ChunkOffsetLengths[chunkIdx];
58+
length_ = chunk.GetLength();
59+
blockSize_ = ioStore_->utoc_.Header.CompressionBlockSize;
60+
compBlockFirstIdx_ = chunk.GetOffset() / blockSize_;
61+
compBlockNum_ = (length_ + blockSize_ - 1) / blockSize_;
62+
buffer_ = std::vector<char>(blockSize_, 0);
63+
currentBlock_ = -1;
64+
}
65+
66+
[[nodiscard]] pos_type tell() const override {
67+
return pos_;
68+
}
69+
70+
void seek(pos_type pos) override {
71+
if (pos > length_) {
72+
throw std::out_of_range("AssetFileIStream: Seek out of range.");
73+
}
74+
pos_ = pos;
75+
}
76+
77+
void seek_relative(off_type off) override {
78+
if (off < 0 && static_cast<pos_type>(-off) > pos_) {
79+
throw std::out_of_range("AssetFileIStream: Seek out of range.");
80+
}
81+
if (off > 0 && pos_ + off > length_) {
82+
throw std::out_of_range("AssetFileIStream: Seek out of range.");
83+
}
84+
pos_ += off;
85+
}
86+
87+
[[nodiscard]] pos_type size() const override {
88+
return length_;
89+
}
90+
91+
void read(std::span<std::byte> out) override {
92+
if (pos_ + out.size() > length_) {
93+
throw std::runtime_error("AssetFileIStream: Not enough data to read.");
94+
}
95+
std::byte* dst = out.data();
96+
std::size_t to_read = out.size();
97+
98+
while (to_read > 0) {
99+
uint32_t posBlock = pos_ / blockSize_;
100+
if (posBlock != currentBlock_) {
101+
const uint32_t expectedSize =
102+
(posBlock < compBlockNum_ - 1) ? blockSize_ : length_ - (compBlockNum_ - 1) * blockSize_;
103+
ioStore_->readCompressionBlock(compBlockFirstIdx_ + posBlock, buffer_.data(), expectedSize);
104+
currentBlock_ = static_cast<int32_t>(posBlock);
105+
}
106+
const uint32_t posInBlock = pos_ % blockSize_;
107+
const std::size_t read_size = std::min(static_cast<std::size_t>(blockSize_ - posInBlock), to_read);
108+
std::memcpy(dst, buffer_.data() + posInBlock, read_size);
109+
dst += read_size;
110+
to_read -= read_size;
111+
pos_ += read_size;
112+
}
113+
}
114+
115+
protected:
116+
std::shared_ptr<IoStoreFile> ioStore_;
117+
uint64_t length_;
118+
uint32_t blockSize_;
119+
uint32_t compBlockFirstIdx_;
120+
uint32_t compBlockNum_;
121+
std::vector<char> buffer_;
122+
int32_t currentBlock_;
123+
pos_type pos_;
124+
};
125+
126+
[[nodiscard]] uint32_t getChunkIdx(const std::string& filename) const;
127+
128+
void readCompressionBlock(uint64_t blockIdx, char* dst, uint32_t expectedSize);
129+
50130
FIoStoreTocResource utoc_;
51131
std::unique_ptr<DirectoryIndexReader> dirIndex_;
52132
std::unique_ptr<IStreamArchive> ucasAr_;

‎libsave/src/Pak/AbstractPakFile.cpp‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
#include "Pak/AbstractPakFile.h"
22

3+
std::unique_ptr<SatisfactorySave::IStream> SatisfactorySave::AbstractPakFile::getAssetFileStream(
4+
const std::string& filename) {
5+
return std::make_unique<MemoryBufferIStream>(std::move(readAssetFileContent(filename)));
6+
}
7+
38
SatisfactorySave::AssetFile SatisfactorySave::AbstractPakFile::readAsset(const std::string& filename) {
49
std::string filenameBase;
510
if (filename.ends_with(".uasset")) {
@@ -11,8 +16,8 @@ SatisfactorySave::AssetFile SatisfactorySave::AbstractPakFile::readAsset(const s
1116
}
1217
const std::string filenameUbulk = filenameBase + "ubulk";
1318

14-
auto uassetFile = readAssetFileContent(filename);
15-
auto ubulkFile = containsAssetFilename(filenameUbulk) ? readAssetFileContent(filenameUbulk) : std::vector<char>();
19+
auto uassetFile = getAssetFileStream(filename);
20+
auto ubulkFile = containsAssetFilename(filenameUbulk) ? getAssetFileStream(filenameUbulk) : nullptr;
1621

1722
return AssetFile(pakManager_, std::move(uassetFile), std::move(ubulkFile));
1823
}

‎libsave/src/Pak/AssetFile.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@
77
#include "IO/Archive/IStreamArchive.h"
88
#include "Pak/PakManager.h"
99

10-
SatisfactorySave::AssetFile::AssetFile(std::shared_ptr<PakManager> pakManager, std::vector<char>&& uassetData,
11-
std::vector<char>&& ubulkData)
12-
: IStreamArchive(std::move(uassetData)),
10+
SatisfactorySave::AssetFile::AssetFile(std::shared_ptr<PakManager> pakManager, std::unique_ptr<IStream> uassetStream,
11+
std::unique_ptr<IStream> ubulkStream)
12+
: IStreamArchive(std::move(uassetStream)),
1313
pakManager_(std::move(pakManager)) {
14-
if (!ubulkData.empty()) {
15-
ubulk_ar_ = std::make_unique<IStreamArchive>(std::move(ubulkData));
14+
if (ubulkStream != nullptr) {
15+
ubulk_ar_ = std::make_unique<IStreamArchive>(std::move(ubulkStream));
1616
}
1717

1818
// Check old asset format

‎libsave/src/Pak/IoStoreFile.cpp‎

Lines changed: 43 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -84,17 +84,13 @@ std::vector<std::string> SatisfactorySave::IoStoreFile::getAllAssetFilenames() c
8484
}
8585

8686
std::vector<char> SatisfactorySave::IoStoreFile::readAssetFileContent(const std::string& filename) {
87-
if (dirIndex_ == nullptr) {
88-
throw std::runtime_error("Missing directory index!");
89-
}
90-
if (!dirIndex_->directoryEntries().contains(filename)) {
91-
throw std::runtime_error("Asset file not found in utoc: " + filename);
92-
}
93-
94-
// UserData of directory index file entry is index to chunk data
95-
uint32_t chunkIdx = dirIndex_->directoryEntries().at(filename);
87+
return readChunkContent(getChunkIdx(filename));
88+
}
9689

97-
return readChunkContent(chunkIdx);
90+
std::unique_ptr<SatisfactorySave::IStream> SatisfactorySave::IoStoreFile::getAssetFileStream(
91+
const std::string& filename) {
92+
return std::make_unique<AssetFileIStream>(std::static_pointer_cast<IoStoreFile>(shared_from_this()),
93+
getChunkIdx(filename));
9894
}
9995

10096
std::vector<char> SatisfactorySave::IoStoreFile::readChunkContent(std::size_t chunkIdx) {
@@ -111,30 +107,46 @@ std::vector<char> SatisfactorySave::IoStoreFile::readChunkContent(std::size_t ch
111107
// Allocate output
112108
std::vector<char> buf(chunk.GetLength());
113109

114-
auto& ar = *ucasAr_;
115-
116110
// Loop over compression blocks
117111
for (uint32_t i = 0; i < compBlockNum; i++) {
118-
const auto& compBlock = utoc_.CompressionBlocks[compBlockIdx + i];
119-
120-
// Validate that uncompressed size of all compression blocks (except the last one) is block size.
121-
if (i < compBlockNum - 1 && compBlock.GetUncompressedSize() != blockSize) {
122-
throw std::runtime_error("Unexpected uncompressed block size");
123-
}
124-
125-
ar.seek(compBlock.GetOffset());
126-
const auto compMethod = utoc_.CompressionMethods.at(compBlock.GetCompressionMethodIndex());
127-
if (compMethod == "None") {
128-
// Uncompressed, copy directly to out buffer.
129-
ar.serializeRaw(buf.data() + i * blockSize, compBlock.GetCompressedSize());
130-
} else if (compMethod == "Oodle") {
131-
const auto comp_buf = ar.read_buffer(compBlock.GetCompressedSize());
132-
oodleDecompress(buf.data() + i * blockSize, compBlock.GetUncompressedSize(), comp_buf.data(),
133-
compBlock.GetCompressedSize());
134-
} else {
135-
throw std::runtime_error("Unknown compression method: " + compMethod);
136-
}
112+
const uint32_t expectedSize =
113+
(i < compBlockNum - 1) ? blockSize : chunk.GetLength() - (compBlockNum - 1) * blockSize;
114+
readCompressionBlock(compBlockIdx + i, buf.data() + i * blockSize, expectedSize);
137115
}
138116

139117
return buf;
140118
}
119+
120+
uint32_t SatisfactorySave::IoStoreFile::getChunkIdx(const std::string& filename) const {
121+
if (dirIndex_ == nullptr) {
122+
throw std::runtime_error("Missing directory index!");
123+
}
124+
if (!dirIndex_->directoryEntries().contains(filename)) {
125+
throw std::runtime_error("Asset file not found in utoc: " + filename);
126+
}
127+
128+
// UserData of directory index file entry is index to chunk data
129+
return dirIndex_->directoryEntries().at(filename);
130+
}
131+
132+
void SatisfactorySave::IoStoreFile::readCompressionBlock(uint64_t blockIdx, char* dst, uint32_t expectedSize) {
133+
auto& ar = *ucasAr_;
134+
const auto& compBlock = utoc_.CompressionBlocks[blockIdx];
135+
136+
// Validate that uncompressed size of all compression blocks (except the last one) is block size.
137+
if (compBlock.GetUncompressedSize() != expectedSize) {
138+
throw std::runtime_error("Unexpected uncompressed block size");
139+
}
140+
141+
ar.seek(compBlock.GetOffset());
142+
const auto compMethod = utoc_.CompressionMethods.at(compBlock.GetCompressionMethodIndex());
143+
if (compMethod == "None") {
144+
// Uncompressed, copy directly to out buffer.
145+
ar.serializeRaw(dst, compBlock.GetCompressedSize());
146+
} else if (compMethod == "Oodle") {
147+
const auto comp_buf = ar.read_buffer(compBlock.GetCompressedSize());
148+
oodleDecompress(dst, compBlock.GetUncompressedSize(), comp_buf.data(), compBlock.GetCompressedSize());
149+
} else {
150+
throw std::runtime_error("Unknown compression method: " + compMethod);
151+
}
152+
}

0 commit comments

Comments
 (0)