feat: Enhance Huffman encoding in pzip for improved compression efficiency

- Added new methods for histogram calculation, dynamic header writing, and code generation in HuffmanBitWriter.
- Introduced a new writeBlockHuff method to handle Huffman-only compression for low token counts.
- Implemented quick entropy detection to optimize compression decisions based on data characteristics.
- Updated FastDeflate to utilize the new Huffman encoding features for better performance.

Log: Improve Huffman encoding and compression efficiency in pzip

bug: https://pms.uniontech.com/bug-view-346679.html

Author: dengzhongyuan365-dev

3rdparty/pzip/CMakeLists.txt

@@ -35,11 +35,9 @@ target_link_libraries(pzip_core_lib ${ZLIB_LIBRARIES} Threads::Threads)
 target_compile_features(pzip_core_lib PUBLIC cxx_std_17)
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm")
-    # ARM 架构优化
-    target_compile_options(pzip_core_lib PRIVATE -O3 -ffast-math -funroll-loops)
+    target_compile_options(pzip_core_lib PRIVATE -O3 -funroll-loops)
 else()
-    # x86 架构优化
-    target_compile_options(pzip_core_lib PRIVATE -O3 -march=native -ffast-math -funroll-loops)
+    target_compile_options(pzip_core_lib PRIVATE -O3 -mtune=generic -funroll-loops)
 endif()
 
 add_executable(pzip-bin cmd/pzip_main.cpp)

3rdparty/pzip/include/pzip/fast_deflate.h

@@ -17,10 +17,12 @@
 
 #include <cstdint>
 #include <cstring>
+#include <functional>
 #include <vector>
 #include <array>
 #include <algorithm>
 #include <memory>
+#include <utility>
 
 #if defined(__GNUC__) || defined(__clang__)
 #define PZIP_FORCE_INLINE __attribute__((always_inline)) inline
@@ -304,6 +306,13 @@ class HuffmanEncoder {
 class HuffmanBitWriter {
 public:
     explicit HuffmanBitWriter();
+    ~HuffmanBitWriter() = default;
+    
+    HuffmanBitWriter(const HuffmanBitWriter&) = delete;
+    HuffmanBitWriter& operator=(const HuffmanBitWriter&) = delete;
+    
+    HuffmanBitWriter(HuffmanBitWriter&&) = default;
+    HuffmanBitWriter& operator=(HuffmanBitWriter&&) = default;
 
     void reset();
     void flush();
@@ -330,32 +339,46 @@ class HuffmanBitWriter {
 
     void writeBlock(Tokens* tokens, bool eof, const uint8_t* input, size_t inputLen);
     void writeBlockDynamic(Tokens* tokens, bool eof, const uint8_t* input, size_t inputLen, bool sync);
+    void writeBlockHuff(bool eof, const uint8_t* input, size_t inputLen, bool sync);
 
     void writeTokens(const Token* tokens, size_t n, const HCode* leCodes, const HCode* oeCodes);
 
     const std::vector<uint8_t>& data() const { return output_; }
     std::vector<uint8_t>& data() { return output_; }
 
+    void setLogNewTablePenalty(int penalty) { logNewTablePenalty_ = penalty; }
+    
 private:
     void writeOutBits();
     void indexTokens(Tokens* t, bool alwaysEOB);
     void generate();
     int extraBitSize();
     int fixedSize(int extraBits);
     int storedSize(const uint8_t* input, size_t len, bool* storable);
+    void histogram(const uint8_t* input, size_t len);
+    std::pair<int, int> headerSize();
+    void generateCodegen(int numLiterals, int numOffsets, HuffmanEncoder* litEnc, HuffmanEncoder* offEnc);
+    int codegens();
+    void writeDynamicHeader(int numLiterals, int numOffsets, int numCodegens, bool isEof);
 
     std::vector<uint8_t> output_;
     uint64_t bits_ = 0;
     uint8_t nbits_ = 0;
     uint8_t nbytes_ = 0;
     int lastHeader_ = 0;
+    bool lastHuffMan_ = false;
+    int logNewTablePenalty_ = 7;
 
     std::array<uint8_t, 256 + 8> bytes_;
     std::array<uint16_t, LENGTH_CODES_START + 32> literalFreq_;
     std::array<uint16_t, 32> offsetFreq_;
+    std::array<uint16_t, 19> codegenFreq_;
+    std::array<uint8_t, LITERAL_COUNT + OFFSET_CODE_COUNT + 1> codegen_;
 
     std::unique_ptr<HuffmanEncoder> literalEncoding_;
     std::unique_ptr<HuffmanEncoder> offsetEncoding_;
+    std::unique_ptr<HuffmanEncoder> tmpLitEncoding_;
+    std::unique_ptr<HuffmanEncoder> codegenEncoding_;
 };
 
 // ============================================================================
@@ -375,9 +398,13 @@ class FastGen {
     FastGen() : cur_(MAX_STORE_BLOCK_SIZE) {
         hist_.reserve(ALLOC_HISTORY);
     }
+    virtual ~FastGen() = default;
 
     int32_t addBlock(const uint8_t* src, size_t len);
-    void reset();
+    virtual void reset();
+    
+    // 纯虚函数，由子类实现
+    virtual void encode(Tokens* dst, const uint8_t* src, size_t len) = 0;
 
 protected:
     std::vector<uint8_t> hist_;
@@ -392,8 +419,8 @@ class FastEncL1 : public FastGen {
 public:
     FastEncL1() { table_.fill({}); }
 
-    void encode(Tokens* dst, const uint8_t* src, size_t len);
-    void reset();
+    void encode(Tokens* dst, const uint8_t* src, size_t len) override;
+    void reset() override;
 
 private:
     std::array<TableEntry, TABLE_SIZE> table_;
@@ -410,8 +437,8 @@ class FastEncL4 : public FastGen {
         bTable_.fill({});
     }
 
-    void encode(Tokens* dst, const uint8_t* src, size_t len);
-    void reset();
+    void encode(Tokens* dst, const uint8_t* src, size_t len) override;
+    void reset() override;
 
 private:
     std::array<TableEntry, TABLE_SIZE> table_;
@@ -448,22 +475,46 @@ size_t deflateCompress(const uint8_t* input, size_t inputSize,
                        CompressionLevel level = CompressionLevel::DefaultCompression);
 
 // ============================================================================
-// DeflateStream
+// FlateWriter - 流式压缩器（参照 Go klauspost/compress flate.Writer）
 // ============================================================================
 
-class DeflateStream {
+// 输出回调类型（参照 Go io.Writer）
+using WriteFunc = std::function<void(const uint8_t*, size_t)>;
+
+class FlateWriter {
 public:
-    explicit DeflateStream(CompressionLevel level = CompressionLevel::DefaultCompression);
-    ~DeflateStream();
+    // 接收输出目标（参照 Go flate.NewWriter(w io.Writer, level int)）
+    explicit FlateWriter(WriteFunc output, CompressionLevel level = CompressionLevel::BestSpeed);
+    ~FlateWriter() = default;
 
+    // 流式写入数据（参照 Go compressor.write）
     size_t write(const uint8_t* data, size_t size);
-    size_t finish(std::vector<uint8_t>& output);
-    void reset();
+    
+    // 完成压缩（参照 Go compressor.Close）
+    void close();
+    
+    // 重置并设置新的输出目标（参照 Go compressor.Reset）
+    void reset(WriteFunc output);
 
 private:
-    std::unique_ptr<FastDeflate> deflate_;
-    std::vector<uint8_t> buffer_;
-    static constexpr size_t BUFFER_SIZE = 128 * 1024;
+    void storeFast();
+    size_t fillBlock(const uint8_t* data, size_t size);
+    void flushOutput();
+    
+    // 获取当前使用的编码器
+    FastGen* encoder() const { return useL1_ ? static_cast<FastGen*>(encoderL1_.get()) 
+                                             : static_cast<FastGen*>(encoderL4_.get()); }
+    
+    WriteFunc output_;
+    std::vector<uint8_t> window_;
+    size_t windowEnd_ = 0;
+    
+    CompressionLevel level_;
+    bool useL1_ = true;  // Level 1-3 使用 L1，Level 4+ 使用 L4
+    std::unique_ptr<FastEncL1> encoderL1_;
+    std::unique_ptr<FastEncL4> encoderL4_;
+    std::unique_ptr<HuffmanBitWriter> writer_;
+    Tokens tokens_;
 };
 
 } // namespace pzip

3rdparty/pzip/src/archiver.cpp

@@ -180,75 +180,52 @@ Error Archiver::compressFile(FileTask* task) {
 }
 
 Error Archiver::compress(FileTask* task) {
-    // 目录不需要压缩
     if (fs::is_directory(task->status)) {
         return Error();
     }
 
-    // 打开源文件
     std::ifstream file(task->path, std::ios::binary);
     if (!file.is_open()) {
         return Error(ErrorCode::FILE_OPEN_ERROR, "Cannot open file: " + task->path.string());
     }
 
-    // 读取整个文件到内存
-    std::vector<uint8_t> fileData(task->fileSize);
-    file.read(reinterpret_cast<char*>(fileData.data()), task->fileSize);
-    if (static_cast<size_t>(file.gcount()) != task->fileSize) {
-        return Error(ErrorCode::FILE_READ_ERROR, "Failed to read file: " + task->path.string());
+    FlateWriter writer([task](const uint8_t* data, size_t size) {
+        task->write(data, size);
+    });
+    
+    constexpr size_t BUFFER_SIZE = 32 * 1024;
+    std::vector<uint8_t> buf(BUFFER_SIZE);
+    uint32_t crc = 0;
+    uint64_t totalBytesRead = 0;
+    
+    while (file.good() && !file.eof()) {
+        file.read(reinterpret_cast<char*>(buf.data()), buf.size());
+        auto bytesRead = file.gcount();
+        if (bytesRead > 0) {
+            crc = ::crc32(crc, buf.data(), bytesRead);
+            writer.write(buf.data(), bytesRead);
+            totalBytesRead += bytesRead;
+        }
     }
-    file.close();
-    
-#ifdef USE_LIBDEFLATE
-    // 使用 libdeflate（高性能）
-    // 注意：libdeflate level 1 最快，level 12 压缩率最高
-    // 默认使用 level 1（最快），用户可以通过 -6 等参数调整
-    task->header.crc32 = libdeflate_crc32(0, fileData.data(), fileData.size());
 
-    int level = options_.compressionLevel;
-    if (level < 1 || level > 12) level = 1;  // 默认使用最快级别
-    
-    struct libdeflate_compressor* compressor = libdeflate_alloc_compressor(level);
-    if (!compressor) {
-        return Error(ErrorCode::COMPRESSION_ERROR, "Failed to create compressor");
+    // 检查 I/O 错误（bad() 表示严重错误，fail() 在 eof 时也会设置所以需要排除）
+    if (file.bad()) {
+        file.close();
+        return Error(ErrorCode::FILE_READ_ERROR, "I/O error reading file: " + task->path.string());
     }
 
-    size_t maxCompressedSize = libdeflate_deflate_compress_bound(compressor, fileData.size());
-    std::vector<uint8_t> compressed(maxCompressedSize);
-    
-    size_t compressedSize = libdeflate_deflate_compress(
-        compressor, 
-        fileData.data(), fileData.size(),
-        compressed.data(), compressed.size()
-    );
-    
-    libdeflate_free_compressor(compressor);
-    
-    if (compressedSize == 0 && !fileData.empty()) {
-        return Error(ErrorCode::COMPRESSION_ERROR, "Compression failed");
+    // 检查是否读取了完整文件（防止静默截断）
+    if (totalBytesRead != task->fileSize) {
+        file.close();
+        return Error(ErrorCode::FILE_READ_ERROR, 
+            "Short read: expected " + std::to_string(task->fileSize) + 
+            " bytes, got " + std::to_string(totalBytesRead) + " for: " + task->path.string());
     }
 
-    task->write(compressed.data(), compressedSize);
-#else
-    // 使用内置压缩器 - 使用 thread_local 避免每次创建新对象
-    task->header.crc32 = ::crc32(0L, fileData.data(), fileData.size());
-    
-    // thread_local 压缩器（使用最快级别）和输出缓冲区，避免重复分配
-    thread_local FastDeflate deflate(CompressionLevel::BestSpeed);
-    thread_local std::vector<uint8_t> compressed;
-    
-    // 重置压缩器状态并清空缓冲区
-    deflate.reset();
-    compressed.clear();
-    
-    size_t compressedSize = deflate.compress(fileData.data(), fileData.size(), compressed);
-    
-    if (compressedSize == 0 && !fileData.empty()) {
-        return Error(ErrorCode::COMPRESSION_ERROR, "Compression failed");
-    }
+    file.close();
 
-    task->write(compressed.data(), compressed.size());
-#endif
+    task->header.crc32 = crc;
+    writer.close();
 
     return Error();
 }