Skip to content

Commit e2a38c9

Browse files
dengzhongyuan365-devdeepin-bot[bot]
dengzhongyuan365-dev
authored and
deepin-bot[bot]
committed
fix: Enhance compression options and ARM CRC32 support
- Updated the CliPzipPlugin to allow dynamic compression levels (0-9) instead of a fixed level. - Introduced a 'streamFromSource' option in FileTask for direct reading from source files in Store mode. - Improved CRC32 calculation for ARM architecture with hardware support detection. - Adjusted ZipWriter to ensure accurate header updates after data writing. This enhances flexibility in compression settings and optimizes performance on ARM platforms. bug: https://pms.uniontech.com/bug-view-310401.html
1 parent 975739d commit e2a38c9

File tree

7 files changed

+151
-18
lines changed

7 files changed

+151
-18
lines changed

‎3rdparty/clipzipplugin/clipzipplugin.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,8 +190,12 @@ PluginFinishType CliPzipPlugin::addFiles(const QList<FileEntry> &files, const Co
190190
// 静默模式
191191
arguments << "-q";
192192

193-
arguments << "-l" << "1";
194-
Q_UNUSED(options.iCompressionLevel);
193+
// 压缩等级：0=Store(不压缩)，1-9=deflate 压缩级别
194+
int level = options.iCompressionLevel;
195+
if (level < 0 || level > 9) {
196+
level = 1;
197+
}
198+
arguments << "-l" << QString::number(level);
195199

196200
// 线程数：只有大于1时才指定，否则让 pzip 自动使用全部 CPU 核心
197201
if (options.iCPUTheadNum > 1) {

‎src/pzip/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ target_link_libraries(pzip_core_lib ${ZLIB_LIBRARIES} Threads::Threads)
3535
target_compile_features(pzip_core_lib PUBLIC cxx_std_17)
3636

3737
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm")
38-
target_compile_options(pzip_core_lib PRIVATE -O3 -funroll-loops)
38+
target_compile_options(pzip_core_lib PRIVATE -O3 -funroll-loops -march=armv8-a+crc)
3939
else()
4040
target_compile_options(pzip_core_lib PRIVATE -O3 -mtune=generic -funroll-loops)
4141
endif()

‎src/pzip/include/pzip/file_task.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ class FileTask {
108108
ZipFileHeader header; // ZIP 头信息
109109
bool isSymlink = false; // 是否是符号链接
110110
std::string symlinkTarget; // 符号链接目标路径
111+
bool streamFromSource = false; // Store 模式：写入时直接从源文件读取，跳过缓冲
111112

112113
// 压缩器（由 Archiver 管理）
113114
z_stream* compressor = nullptr;

‎src/pzip/src/archiver.cpp‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "pzip/archiver.h"
77
#include "pzip/fast_deflate.h"
88
#include "pzip/utils.h"
9+
#include <algorithm>
910
#include <fstream>
1011
#include <cstring>
1112
#include <sys/stat.h>
@@ -196,15 +197,28 @@ Error Archiver::compress(FileTask* task) {
196197
return Error(ErrorCode::FILE_OPEN_ERROR, "Cannot open file: " + task->path.string());
197198
}
198199

199-
FlateWriter writer([task](const uint8_t* data, size_t size) {
200-
task->write(data, size);
201-
});
200+
const bool useStore = (options_.compressionLevel == 0);
202201

203202
constexpr size_t BUFFER_SIZE = 32 * 1024;
204203
std::vector<uint8_t> buf(BUFFER_SIZE);
205204
uint32_t crc = 0;
206205
uint64_t totalBytesRead = 0;
207206

207+
if (useStore) {
208+
// Store 模式：跳过读文件，写入 ZIP 时直接从源文件流式读取并计算 CRC
209+
file.close();
210+
task->streamFromSource = true;
211+
return Error();
212+
}
213+
214+
// Deflate 模式：使用 FlateWriter 按指定等级压缩
215+
auto level = static_cast<CompressionLevel>(
216+
std::clamp(options_.compressionLevel, 1, 9));
217+
218+
FlateWriter writer([task](const uint8_t* data, size_t size) {
219+
task->write(data, size);
220+
}, level);
221+
208222
while (file.good() && !file.eof()) {
209223
file.read(reinterpret_cast<char*>(buf.data()), buf.size());
210224
auto bytesRead = file.gcount();
@@ -215,13 +229,13 @@ Error Archiver::compress(FileTask* task) {
215229
}
216230
}
217231

218-
// 检查 I/O 错误（bad() 表示严重错误，fail() 在 eof 时也会设置所以需要排除）
232+
writer.close();
233+
219234
if (file.bad()) {
220235
file.close();
221236
return Error(ErrorCode::FILE_READ_ERROR, "I/O error reading file: " + task->path.string());
222237
}
223238

224-
// 检查是否读取了完整文件（防止静默截断）
225239
if (totalBytesRead != task->fileSize) {
226240
file.close();
227241
return Error(ErrorCode::FILE_READ_ERROR,
@@ -232,7 +246,6 @@ Error Archiver::compress(FileTask* task) {
232246
file.close();
233247

234248
task->header.crc32 = crc;
235-
writer.close();
236249

237250
return Error();
238251
}
@@ -286,6 +299,12 @@ void Archiver::populateHeader(FileTask* task) {
286299
h.compressedSize = task->symlinkTarget.size();
287300
// 设置 Unix 符号链接属性
288301
h.externalAttr = static_cast<uint32_t>(S_IFLNK | 0777) << 16;
302+
} else if (options_.compressionLevel == 0) {
303+
// Store 模式：不压缩，CRC 在写入阶段边读边算，用 DATA_DESCRIPTOR 后置
304+
h.method = ZIP_METHOD_STORE;
305+
h.flags |= ZIP_FLAG_DATA_DESCRIPTOR;
306+
h.uncompressedSize = task->fileSize;
307+
h.compressedSize = task->fileSize;
289308
} else {
290309
h.method = ZIP_METHOD_DEFLATE;
291310
h.flags |= ZIP_FLAG_DATA_DESCRIPTOR;

‎src/pzip/src/file_task.cpp‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ FileTask::FileTask(FileTask&& other) noexcept
3636
, status(other.status)
3737
, fileSize(other.fileSize)
3838
, header(std::move(other.header))
39+
, isSymlink(other.isSymlink)
40+
, symlinkTarget(std::move(other.symlinkTarget))
41+
, streamFromSource(other.streamFromSource)
3942
, compressor(other.compressor)
4043
, buffer_(std::move(other.buffer_))
4144
, bufferUsed_(other.bufferUsed_)
@@ -44,6 +47,8 @@ FileTask::FileTask(FileTask&& other) noexcept
4447
, written_(other.written_)
4548
{
4649
other.compressor = nullptr;
50+
other.isSymlink = false;
51+
other.streamFromSource = false;
4752
other.bufferUsed_ = 0;
4853
other.written_ = 0;
4954
}
@@ -54,6 +59,9 @@ FileTask& FileTask::operator=(FileTask&& other) noexcept {
5459
status = other.status;
5560
fileSize = other.fileSize;
5661
header = std::move(other.header);
62+
isSymlink = other.isSymlink;
63+
symlinkTarget = std::move(other.symlinkTarget);
64+
streamFromSource = other.streamFromSource;
5765
compressor = other.compressor;
5866
buffer_ = std::move(other.buffer_);
5967
bufferUsed_ = other.bufferUsed_;
@@ -62,6 +70,8 @@ FileTask& FileTask::operator=(FileTask&& other) noexcept {
6270
written_ = other.written_;
6371

6472
other.compressor = nullptr;
73+
other.isSymlink = false;
74+
other.streamFromSource = false;
6575
other.bufferUsed_ = 0;
6676
other.written_ = 0;
6777
}
@@ -84,6 +94,7 @@ Error FileTask::reset(const fs::path& filePath, const fs::path& relativeTo) {
8494
written_ = 0;
8595
isSymlink = false;
8696
symlinkTarget.clear();
97+
streamFromSource = false;
8798

8899
// 设置新文件信息
89100
path = filePath;
@@ -184,26 +195,52 @@ size_t FileTask::write(const uint8_t* data, size_t size) {
184195
);
185196

186197
if (!overflow_->is_open()) {
198+
overflow_.reset();
187199
return totalWritten;
188200
}
189201
}
190202

191-
overflow_->write(reinterpret_cast<const char*>(data), size);
192-
totalWritten += size;
203+
if (overflow_ && overflow_->is_open()) {
204+
overflow_->write(reinterpret_cast<const char*>(data), size);
205+
if (overflow_->good()) {
206+
totalWritten += size;
207+
}
208+
}
193209
}
194210

195211
written_ += totalWritten;
196212
return totalWritten;
197213
}
198214

199215
void FileTask::readCompressedData(std::function<void(const uint8_t*, size_t)> callback) {
216+
// Store 模式：直接从源文件流式读取，边读边算 CRC
217+
if (streamFromSource) {
218+
std::ifstream file(path, std::ios::binary);
219+
if (!file.is_open()) return;
220+
221+
uint32_t crc = 0;
222+
std::vector<uint8_t> readBuf(READ_BUFFER_SIZE);
223+
while (file.good() && !file.eof()) {
224+
file.read(reinterpret_cast<char*>(readBuf.data()), readBuf.size());
225+
auto bytesRead = file.gcount();
226+
if (bytesRead > 0) {
227+
crc = utils::crc32Update(crc, readBuf.data(), bytesRead);
228+
callback(readBuf.data(), bytesRead);
229+
}
230+
}
231+
header.crc32 = crc;
232+
return;
233+
}
234+
200235
// 先读取内存缓冲区
201236
if (bufferUsed_ > 0) {
202237
callback(buffer_.data(), bufferUsed_);
203238
}
204239

205240
// 再读取溢出文件
206-
if (overflow_) {
241+
if (overflow_ && overflow_->is_open()) {
242+
overflow_->flush();
243+
overflow_->clear();
207244
overflow_->seekg(0, std::ios::beg);
208245

209246
std::vector<uint8_t> readBuf(READ_BUFFER_SIZE);

‎src/pzip/src/utils.cpp‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,17 @@
1212
#include <cstring>
1313
#include <random>
1414

15+
#if defined(__aarch64__)
16+
#include <sys/auxv.h>
17+
#include <asm/hwcap.h>
18+
#ifdef __ARM_FEATURE_CRC32
19+
#include <arm_acle.h>
20+
#define PZIP_HAS_ARM_CRC32_COMPILE 1
21+
#else
22+
#define PZIP_HAS_ARM_CRC32_COMPILE 0
23+
#endif
24+
#endif
25+
1526
namespace pzip {
1627
namespace utils {
1728

@@ -97,11 +108,70 @@ fs::path fromZipPath(const std::string& zipPath) {
97108
return fs::path(result);
98109
}
99110

111+
#if defined(__aarch64__) && PZIP_HAS_ARM_CRC32_COMPILE
112+
static uint32_t crc32_arm_hw(uint32_t crc, const uint8_t* data, size_t size) {
113+
uint32_t c = crc ^ 0xFFFFFFFF;
114+
115+
// 逐字节对齐到 8 字节边界
116+
while (size > 0 && (reinterpret_cast<uintptr_t>(data) & 7)) {
117+
c = __crc32b(c, *data++);
118+
size--;
119+
}
120+
121+
// 每次处理 8 字节
122+
while (size >= 64) {
123+
c = __crc32d(c, *reinterpret_cast<const uint64_t*>(data));
124+
c = __crc32d(c, *reinterpret_cast<const uint64_t*>(data + 8));
125+
c = __crc32d(c, *reinterpret_cast<const uint64_t*>(data + 16));
126+
c = __crc32d(c, *reinterpret_cast<const uint64_t*>(data + 24));
127+
c = __crc32d(c, *reinterpret_cast<const uint64_t*>(data + 32));
128+
c = __crc32d(c, *reinterpret_cast<const uint64_t*>(data + 40));
129+
c = __crc32d(c, *reinterpret_cast<const uint64_t*>(data + 48));
130+
c = __crc32d(c, *reinterpret_cast<const uint64_t*>(data + 56));
131+
data += 64;
132+
size -= 64;
133+
}
134+
135+
while (size >= 8) {
136+
c = __crc32d(c, *reinterpret_cast<const uint64_t*>(data));
137+
data += 8;
138+
size -= 8;
139+
}
140+
141+
while (size > 0) {
142+
c = __crc32b(c, *data++);
143+
size--;
144+
}
145+
146+
return c ^ 0xFFFFFFFF;
147+
}
148+
149+
static bool hasArmCrc32() {
150+
static bool checked = false;
151+
static bool supported = false;
152+
if (!checked) {
153+
supported = (getauxval(AT_HWCAP) & HWCAP_CRC32) != 0;
154+
checked = true;
155+
}
156+
return supported;
157+
}
158+
#endif
159+
100160
uint32_t crc32(const uint8_t* data, size_t size) {
161+
#if defined(__aarch64__) && PZIP_HAS_ARM_CRC32_COMPILE
162+
if (hasArmCrc32()) {
163+
return crc32_arm_hw(0, data, size);
164+
}
165+
#endif
101166
return ::crc32(0L, data, size);
102167
}
103168

104169
uint32_t crc32Update(uint32_t crc, const uint8_t* data, size_t size) {
170+
#if defined(__aarch64__) && PZIP_HAS_ARM_CRC32_COMPILE
171+
if (hasArmCrc32()) {
172+
return crc32_arm_hw(crc, data, size);
173+
}
174+
#endif
105175
return ::crc32(crc, data, size);
106176
}
107177

‎src/pzip/src/zip_writer.cpp‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -257,16 +257,14 @@ Error ZipWriter::createRaw(const ZipFileHeader& header,
257257
return Error(ErrorCode::FILE_OPEN_ERROR, "File not open");
258258
}
259259

260-
// 保存本地文件头偏移
261-
CentralDirEntry entry;
262-
entry.header = header;
263-
entry.localHeaderOffset = currentOffset_;
260+
// 记录本地文件头偏移
261+
uint64_t localHeaderOffset = currentOffset_;
264262

265263
// 写入本地文件头
266264
Error err = writeLocalFileHeader(header);
267265
if (err) return err;
268266

269-
// 写入压缩数据
267+
// 写入压缩数据（dataProvider 可能会更新 header 中的 CRC 等字段）
270268
dataProvider([this](const uint8_t* data, size_t size) {
271269
file_.write(reinterpret_cast<const char*>(data), size);
272270
currentOffset_ += size;
@@ -276,12 +274,16 @@ Error ZipWriter::createRaw(const ZipFileHeader& header,
276274
return Error(ErrorCode::FILE_WRITE_ERROR, "Failed to write compressed data");
277275
}
278276

279-
// 如果使用数据描述符，写入它
277+
// 如果使用数据描述符，写入它（此时 header.crc32 已在 dataProvider 中更新）
280278
if (header.flags & ZIP_FLAG_DATA_DESCRIPTOR) {
281279
err = writeDataDescriptor(header);
282280
if (err) return err;
283281
}
284282

283+
// dataProvider 运行后再拷贝 header，确保 CRC 等字段是最新值
284+
CentralDirEntry entry;
285+
entry.header = header;
286+
entry.localHeaderOffset = localHeaderOffset;
285287
centralDir_.push_back(entry);
286288
return Error();
287289
}

0 commit comments

Comments
 (0)