From 3c8a87e618305c5ee8d7d37ba90e0a32cb06f573 Mon Sep 17 00:00:00 2001 From: dengzhongyuan Date: Tue, 3 Mar 2026 16:40:46 +0800 Subject: [PATCH] fix: Enhance compression options and ARM CRC32 support - Updated the CliPzipPlugin to allow dynamic compression levels (0-9) instead of a fixed level. - Introduced a 'streamFromSource' option in FileTask for direct reading from source files in Store mode. - Improved CRC32 calculation for ARM architecture with hardware support detection. - Adjusted ZipWriter to ensure accurate header updates after data writing. This enhances flexibility in compression settings and optimizes performance on ARM platforms. bug: https://pms.uniontech.com/bug-view-310401.html --- 3rdparty/clipzipplugin/clipzipplugin.cpp | 8 ++- src/pzip/CMakeLists.txt | 2 +- src/pzip/include/pzip/file_task.h | 1 + src/pzip/src/archiver.cpp | 31 +++++++++-- src/pzip/src/file_task.cpp | 43 ++++++++++++++- src/pzip/src/utils.cpp | 70 ++++++++++++++++++++++++ src/pzip/src/zip_writer.cpp | 14 +++-- 7 files changed, 151 insertions(+), 18 deletions(-) diff --git a/3rdparty/clipzipplugin/clipzipplugin.cpp b/3rdparty/clipzipplugin/clipzipplugin.cpp index 985b5b21..ab362712 100644 --- a/3rdparty/clipzipplugin/clipzipplugin.cpp +++ b/3rdparty/clipzipplugin/clipzipplugin.cpp @@ -190,8 +190,12 @@ PluginFinishType CliPzipPlugin::addFiles(const QList &files, const Co // 静默模式 arguments << "-q"; - arguments << "-l" << "1"; - Q_UNUSED(options.iCompressionLevel); + // 压缩等级：0=Store(不压缩)，1-9=deflate 压缩级别 + int level = options.iCompressionLevel; + if (level < 0 || level > 9) { + level = 1; + } + arguments << "-l" << QString::number(level); // 线程数：只有大于1时才指定，否则让 pzip 自动使用全部 CPU 核心 if (options.iCPUTheadNum > 1) { diff --git a/src/pzip/CMakeLists.txt b/src/pzip/CMakeLists.txt index d13c82a8..95787bb3 100644 --- a/src/pzip/CMakeLists.txt +++ b/src/pzip/CMakeLists.txt @@ -35,7 +35,7 @@ target_link_libraries(pzip_core_lib ${ZLIB_LIBRARIES} Threads::Threads) target_compile_features(pzip_core_lib PUBLIC cxx_std_17) if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm") - target_compile_options(pzip_core_lib PRIVATE -O3 -funroll-loops) + target_compile_options(pzip_core_lib PRIVATE -O3 -funroll-loops -march=armv8-a+crc) else() target_compile_options(pzip_core_lib PRIVATE -O3 -mtune=generic -funroll-loops) endif() diff --git a/src/pzip/include/pzip/file_task.h b/src/pzip/include/pzip/file_task.h index 3487481e..10994aee 100644 --- a/src/pzip/include/pzip/file_task.h +++ b/src/pzip/include/pzip/file_task.h @@ -108,6 +108,7 @@ class FileTask { ZipFileHeader header; // ZIP 头信息 bool isSymlink = false; // 是否是符号链接 std::string symlinkTarget; // 符号链接目标路径 + bool streamFromSource = false; // Store 模式：写入时直接从源文件读取，跳过缓冲 // 压缩器（由 Archiver 管理） z_stream* compressor = nullptr; diff --git a/src/pzip/src/archiver.cpp b/src/pzip/src/archiver.cpp index 226e4978..8afadaf1 100644 --- a/src/pzip/src/archiver.cpp +++ b/src/pzip/src/archiver.cpp @@ -6,6 +6,7 @@ #include "pzip/archiver.h" #include "pzip/fast_deflate.h" #include "pzip/utils.h" +#include #include #include #include @@ -196,15 +197,28 @@ Error Archiver::compress(FileTask* task) { return Error(ErrorCode::FILE_OPEN_ERROR, "Cannot open file: " + task->path.string()); } - FlateWriter writer([task](const uint8_t* data, size_t size) { - task->write(data, size); - }); + const bool useStore = (options_.compressionLevel == 0); constexpr size_t BUFFER_SIZE = 32 * 1024; std::vector buf(BUFFER_SIZE); uint32_t crc = 0; uint64_t totalBytesRead = 0; + if (useStore) { + // Store 模式：跳过读文件，写入 ZIP 时直接从源文件流式读取并计算 CRC + file.close(); + task->streamFromSource = true; + return Error(); + } + + // Deflate 模式：使用 FlateWriter 按指定等级压缩 + auto level = static_cast( + std::clamp(options_.compressionLevel, 1, 9)); + + FlateWriter writer([task](const uint8_t* data, size_t size) { + task->write(data, size); + }, level); + while (file.good() && !file.eof()) { file.read(reinterpret_cast(buf.data()), buf.size()); auto bytesRead = file.gcount(); @@ -215,13 +229,13 @@ Error Archiver::compress(FileTask* task) { } } - // 检查 I/O 错误（bad() 表示严重错误，fail() 在 eof 时也会设置所以需要排除） + writer.close(); + if (file.bad()) { file.close(); return Error(ErrorCode::FILE_READ_ERROR, "I/O error reading file: " + task->path.string()); } - // 检查是否读取了完整文件（防止静默截断） if (totalBytesRead != task->fileSize) { file.close(); return Error(ErrorCode::FILE_READ_ERROR, @@ -232,7 +246,6 @@ Error Archiver::compress(FileTask* task) { file.close(); task->header.crc32 = crc; - writer.close(); return Error(); } @@ -286,6 +299,12 @@ void Archiver::populateHeader(FileTask* task) { h.compressedSize = task->symlinkTarget.size(); // 设置 Unix 符号链接属性 h.externalAttr = static_cast(S_IFLNK | 0777) << 16; + } else if (options_.compressionLevel == 0) { + // Store 模式：不压缩，CRC 在写入阶段边读边算，用 DATA_DESCRIPTOR 后置 + h.method = ZIP_METHOD_STORE; + h.flags |= ZIP_FLAG_DATA_DESCRIPTOR; + h.uncompressedSize = task->fileSize; + h.compressedSize = task->fileSize; } else { h.method = ZIP_METHOD_DEFLATE; h.flags |= ZIP_FLAG_DATA_DESCRIPTOR; diff --git a/src/pzip/src/file_task.cpp b/src/pzip/src/file_task.cpp index 49d04f72..fdbbee52 100644 --- a/src/pzip/src/file_task.cpp +++ b/src/pzip/src/file_task.cpp @@ -36,6 +36,9 @@ FileTask::FileTask(FileTask&& other) noexcept , status(other.status) , fileSize(other.fileSize) , header(std::move(other.header)) + , isSymlink(other.isSymlink) + , symlinkTarget(std::move(other.symlinkTarget)) + , streamFromSource(other.streamFromSource) , compressor(other.compressor) , buffer_(std::move(other.buffer_)) , bufferUsed_(other.bufferUsed_) @@ -44,6 +47,8 @@ FileTask::FileTask(FileTask&& other) noexcept , written_(other.written_) { other.compressor = nullptr; + other.isSymlink = false; + other.streamFromSource = false; other.bufferUsed_ = 0; other.written_ = 0; } @@ -54,6 +59,9 @@ FileTask& FileTask::operator=(FileTask&& other) noexcept { status = other.status; fileSize = other.fileSize; header = std::move(other.header); + isSymlink = other.isSymlink; + symlinkTarget = std::move(other.symlinkTarget); + streamFromSource = other.streamFromSource; compressor = other.compressor; buffer_ = std::move(other.buffer_); bufferUsed_ = other.bufferUsed_; @@ -62,6 +70,8 @@ FileTask& FileTask::operator=(FileTask&& other) noexcept { written_ = other.written_; other.compressor = nullptr; + other.isSymlink = false; + other.streamFromSource = false; other.bufferUsed_ = 0; other.written_ = 0; } @@ -84,6 +94,7 @@ Error FileTask::reset(const fs::path& filePath, const fs::path& relativeTo) { written_ = 0; isSymlink = false; symlinkTarget.clear(); + streamFromSource = false; // 设置新文件信息 path = filePath; @@ -184,12 +195,17 @@ size_t FileTask::write(const uint8_t* data, size_t size) { ); if (!overflow_->is_open()) { + overflow_.reset(); return totalWritten; } } - overflow_->write(reinterpret_cast(data), size); - totalWritten += size; + if (overflow_ && overflow_->is_open()) { + overflow_->write(reinterpret_cast(data), size); + if (overflow_->good()) { + totalWritten += size; + } + } } written_ += totalWritten; @@ -197,13 +213,34 @@ size_t FileTask::write(const uint8_t* data, size_t size) { } void FileTask::readCompressedData(std::function callback) { + // Store 模式：直接从源文件流式读取，边读边算 CRC + if (streamFromSource) { + std::ifstream file(path, std::ios::binary); + if (!file.is_open()) return; + + uint32_t crc = 0; + std::vector readBuf(READ_BUFFER_SIZE); + while (file.good() && !file.eof()) { + file.read(reinterpret_cast(readBuf.data()), readBuf.size()); + auto bytesRead = file.gcount(); + if (bytesRead > 0) { + crc = utils::crc32Update(crc, readBuf.data(), bytesRead); + callback(readBuf.data(), bytesRead); + } + } + header.crc32 = crc; + return; + } + // 先读取内存缓冲区 if (bufferUsed_ > 0) { callback(buffer_.data(), bufferUsed_); } // 再读取溢出文件 - if (overflow_) { + if (overflow_ && overflow_->is_open()) { + overflow_->flush(); + overflow_->clear(); overflow_->seekg(0, std::ios::beg); std::vector readBuf(READ_BUFFER_SIZE); diff --git a/src/pzip/src/utils.cpp b/src/pzip/src/utils.cpp index 08827d7e..6d5b95c5 100644 --- a/src/pzip/src/utils.cpp +++ b/src/pzip/src/utils.cpp @@ -12,6 +12,17 @@ #include #include +#if defined(__aarch64__) +#include +#include +#ifdef __ARM_FEATURE_CRC32 +#include +#define PZIP_HAS_ARM_CRC32_COMPILE 1 +#else +#define PZIP_HAS_ARM_CRC32_COMPILE 0 +#endif +#endif + namespace pzip { namespace utils { @@ -97,11 +108,70 @@ fs::path fromZipPath(const std::string& zipPath) { return fs::path(result); } +#if defined(__aarch64__) && PZIP_HAS_ARM_CRC32_COMPILE +static uint32_t crc32_arm_hw(uint32_t crc, const uint8_t* data, size_t size) { + uint32_t c = crc ^ 0xFFFFFFFF; + + // 逐字节对齐到 8 字节边界 + while (size > 0 && (reinterpret_cast(data) & 7)) { + c = __crc32b(c, *data++); + size--; + } + + // 每次处理 8 字节 + while (size >= 64) { + c = __crc32d(c, *reinterpret_cast(data)); + c = __crc32d(c, *reinterpret_cast(data + 8)); + c = __crc32d(c, *reinterpret_cast(data + 16)); + c = __crc32d(c, *reinterpret_cast(data + 24)); + c = __crc32d(c, *reinterpret_cast(data + 32)); + c = __crc32d(c, *reinterpret_cast(data + 40)); + c = __crc32d(c, *reinterpret_cast(data + 48)); + c = __crc32d(c, *reinterpret_cast(data + 56)); + data += 64; + size -= 64; + } + + while (size >= 8) { + c = __crc32d(c, *reinterpret_cast(data)); + data += 8; + size -= 8; + } + + while (size > 0) { + c = __crc32b(c, *data++); + size--; + } + + return c ^ 0xFFFFFFFF; +} + +static bool hasArmCrc32() { + static bool checked = false; + static bool supported = false; + if (!checked) { + supported = (getauxval(AT_HWCAP) & HWCAP_CRC32) != 0; + checked = true; + } + return supported; +} +#endif + uint32_t crc32(const uint8_t* data, size_t size) { +#if defined(__aarch64__) && PZIP_HAS_ARM_CRC32_COMPILE + if (hasArmCrc32()) { + return crc32_arm_hw(0, data, size); + } +#endif return ::crc32(0L, data, size); } uint32_t crc32Update(uint32_t crc, const uint8_t* data, size_t size) { +#if defined(__aarch64__) && PZIP_HAS_ARM_CRC32_COMPILE + if (hasArmCrc32()) { + return crc32_arm_hw(crc, data, size); + } +#endif return ::crc32(crc, data, size); } diff --git a/src/pzip/src/zip_writer.cpp b/src/pzip/src/zip_writer.cpp index 58d9e24a..4c42ce4a 100644 --- a/src/pzip/src/zip_writer.cpp +++ b/src/pzip/src/zip_writer.cpp @@ -257,16 +257,14 @@ Error ZipWriter::createRaw(const ZipFileHeader& header, return Error(ErrorCode::FILE_OPEN_ERROR, "File not open"); } - // 保存本地文件头偏移 - CentralDirEntry entry; - entry.header = header; - entry.localHeaderOffset = currentOffset_; + // 记录本地文件头偏移 + uint64_t localHeaderOffset = currentOffset_; // 写入本地文件头 Error err = writeLocalFileHeader(header); if (err) return err; - // 写入压缩数据 + // 写入压缩数据（dataProvider 可能会更新 header 中的 CRC 等字段） dataProvider([this](const uint8_t* data, size_t size) { file_.write(reinterpret_cast(data), size); currentOffset_ += size; @@ -276,12 +274,16 @@ Error ZipWriter::createRaw(const ZipFileHeader& header, return Error(ErrorCode::FILE_WRITE_ERROR, "Failed to write compressed data"); } - // 如果使用数据描述符，写入它 + // 如果使用数据描述符，写入它（此时 header.crc32 已在 dataProvider 中更新） if (header.flags & ZIP_FLAG_DATA_DESCRIPTOR) { err = writeDataDescriptor(header); if (err) return err; } + // dataProvider 运行后再拷贝 header，确保 CRC 等字段是最新值 + CentralDirEntry entry; + entry.header = header; + entry.localHeaderOffset = localHeaderOffset; centralDir_.push_back(entry); return Error(); }