feat: Enhance Huffman encoding in pzip for improved compression efficiency

- Added new methods for histogram calculation, dynamic header writing, and code generation in HuffmanBitWriter.
- Introduced a new writeBlockHuff method to handle Huffman-only compression for low token counts.
- Implemented quick entropy detection to optimize compression decisions based on data characteristics.
- Updated FastDeflate to utilize the new Huffman encoding features for better performance.

Log: Improve Huffman encoding and compression efficiency in pzip

bug: https://pms.uniontech.com/bug-view-346679.html

Author: dengzhongyuan365-dev

3rdparty/pzip/include/pzip/fast_deflate.h

@@ -17,10 +17,12 @@
 
 #include <cstdint>
 #include <cstring>
+#include <functional>
 #include <vector>
 #include <array>
 #include <algorithm>
 #include <memory>
+#include <utility>
 
 #if defined(__GNUC__) || defined(__clang__)
 #define PZIP_FORCE_INLINE __attribute__((always_inline)) inline
@@ -330,32 +332,46 @@ class HuffmanBitWriter {
 
     void writeBlock(Tokens* tokens, bool eof, const uint8_t* input, size_t inputLen);
     void writeBlockDynamic(Tokens* tokens, bool eof, const uint8_t* input, size_t inputLen, bool sync);
+    void writeBlockHuff(bool eof, const uint8_t* input, size_t inputLen, bool sync);
 
     void writeTokens(const Token* tokens, size_t n, const HCode* leCodes, const HCode* oeCodes);
 
     const std::vector<uint8_t>& data() const { return output_; }
     std::vector<uint8_t>& data() { return output_; }
 
+    void setLogNewTablePenalty(int penalty) { logNewTablePenalty_ = penalty; }
+    
 private:
     void writeOutBits();
     void indexTokens(Tokens* t, bool alwaysEOB);
     void generate();
     int extraBitSize();
     int fixedSize(int extraBits);
     int storedSize(const uint8_t* input, size_t len, bool* storable);
+    void histogram(const uint8_t* input, size_t len);
+    std::pair<int, int> headerSize();
+    void generateCodegen(int numLiterals, int numOffsets, HuffmanEncoder* litEnc, HuffmanEncoder* offEnc);
+    int codegens();
+    void writeDynamicHeader(int numLiterals, int numOffsets, int numCodegens, bool isEof);
 
     std::vector<uint8_t> output_;
     uint64_t bits_ = 0;
     uint8_t nbits_ = 0;
     uint8_t nbytes_ = 0;
     int lastHeader_ = 0;
+    bool lastHuffMan_ = false;
+    int logNewTablePenalty_ = 7;
 
     std::array<uint8_t, 256 + 8> bytes_;
     std::array<uint16_t, LENGTH_CODES_START + 32> literalFreq_;
     std::array<uint16_t, 32> offsetFreq_;
+    std::array<uint16_t, 19> codegenFreq_;
+    std::array<uint8_t, LITERAL_COUNT + OFFSET_CODE_COUNT + 1> codegen_;
 
     std::unique_ptr<HuffmanEncoder> literalEncoding_;
     std::unique_ptr<HuffmanEncoder> offsetEncoding_;
+    std::unique_ptr<HuffmanEncoder> tmpLitEncoding_;
+    std::unique_ptr<HuffmanEncoder> codegenEncoding_;
 };
 
 // ============================================================================
@@ -448,22 +464,39 @@ size_t deflateCompress(const uint8_t* input, size_t inputSize,
                        CompressionLevel level = CompressionLevel::DefaultCompression);
 
 // ============================================================================
-// DeflateStream
+// FlateWriter - 流式压缩器（参照 Go klauspost/compress flate.Writer）
 // ============================================================================
 
-class DeflateStream {
+// 输出回调类型（参照 Go io.Writer）
+using WriteFunc = std::function<void(const uint8_t*, size_t)>;
+
+class FlateWriter {
 public:
-    explicit DeflateStream(CompressionLevel level = CompressionLevel::DefaultCompression);
-    ~DeflateStream();
+    // 接收输出目标（参照 Go flate.NewWriter(w io.Writer, level int)）
+    explicit FlateWriter(WriteFunc output, CompressionLevel level = CompressionLevel::BestSpeed);
+    ~FlateWriter() = default;
 
+    // 流式写入数据（参照 Go compressor.write）
     size_t write(const uint8_t* data, size_t size);
-    size_t finish(std::vector<uint8_t>& output);
-    void reset();
+    
+    // 完成压缩（参照 Go compressor.Close）
+    void close();
+    
+    // 重置并设置新的输出目标（参照 Go compressor.Reset）
+    void reset(WriteFunc output);
 
 private:
-    std::unique_ptr<FastDeflate> deflate_;
-    std::vector<uint8_t> buffer_;
-    static constexpr size_t BUFFER_SIZE = 128 * 1024;
+    void storeFast();
+    size_t fillBlock(const uint8_t* data, size_t size);
+    void flushOutput();
+    
+    WriteFunc output_;
+    std::vector<uint8_t> window_;
+    size_t windowEnd_ = 0;
+    
+    std::unique_ptr<FastEncL1> encoder_;
+    std::unique_ptr<HuffmanBitWriter> writer_;
+    Tokens tokens_;
 };
 
 } // namespace pzip

3rdparty/pzip/src/archiver.cpp

@@ -180,75 +180,35 @@ Error Archiver::compressFile(FileTask* task) {
 }
 
 Error Archiver::compress(FileTask* task) {
-    // 目录不需要压缩
     if (fs::is_directory(task->status)) {
         return Error();
     }
 
-    // 打开源文件
     std::ifstream file(task->path, std::ios::binary);
     if (!file.is_open()) {
         return Error(ErrorCode::FILE_OPEN_ERROR, "Cannot open file: " + task->path.string());
     }
 
-    // 读取整个文件到内存
-    std::vector<uint8_t> fileData(task->fileSize);
-    file.read(reinterpret_cast<char*>(fileData.data()), task->fileSize);
-    if (static_cast<size_t>(file.gcount()) != task->fileSize) {
-        return Error(ErrorCode::FILE_READ_ERROR, "Failed to read file: " + task->path.string());
-    }
-    file.close();
-    
-#ifdef USE_LIBDEFLATE
-    // 使用 libdeflate（高性能）
-    // 注意：libdeflate level 1 最快，level 12 压缩率最高
-    // 默认使用 level 1（最快），用户可以通过 -6 等参数调整
-    task->header.crc32 = libdeflate_crc32(0, fileData.data(), fileData.size());
-    
-    int level = options_.compressionLevel;
-    if (level < 1 || level > 12) level = 1;  // 默认使用最快级别
-    
-    struct libdeflate_compressor* compressor = libdeflate_alloc_compressor(level);
-    if (!compressor) {
-        return Error(ErrorCode::COMPRESSION_ERROR, "Failed to create compressor");
-    }
-    
-    size_t maxCompressedSize = libdeflate_deflate_compress_bound(compressor, fileData.size());
-    std::vector<uint8_t> compressed(maxCompressedSize);
-    
-    size_t compressedSize = libdeflate_deflate_compress(
-        compressor, 
-        fileData.data(), fileData.size(),
-        compressed.data(), compressed.size()
-    );
+    FlateWriter writer([task](const uint8_t* data, size_t size) {
+        task->write(data, size);
+    });
 
-    libdeflate_free_compressor(compressor);
+    constexpr size_t BUFFER_SIZE = 32 * 1024;
+    std::vector<uint8_t> buf(BUFFER_SIZE);
+    uint32_t crc = 0;
 
-    if (compressedSize == 0 && !fileData.empty()) {
-        return Error(ErrorCode::COMPRESSION_ERROR, "Compression failed");
-    }
-    
-    task->write(compressed.data(), compressedSize);
-#else
-    // 使用内置压缩器 - 使用 thread_local 避免每次创建新对象
-    task->header.crc32 = ::crc32(0L, fileData.data(), fileData.size());
-    
-    // thread_local 压缩器（使用最快级别）和输出缓冲区，避免重复分配
-    thread_local FastDeflate deflate(CompressionLevel::BestSpeed);
-    thread_local std::vector<uint8_t> compressed;
-    
-    // 重置压缩器状态并清空缓冲区
-    deflate.reset();
-    compressed.clear();
-    
-    size_t compressedSize = deflate.compress(fileData.data(), fileData.size(), compressed);
-    
-    if (compressedSize == 0 && !fileData.empty()) {
-        return Error(ErrorCode::COMPRESSION_ERROR, "Compression failed");
+    while (file.good() && !file.eof()) {
+        file.read(reinterpret_cast<char*>(buf.data()), buf.size());
+        auto bytesRead = file.gcount();
+        if (bytesRead > 0) {
+            crc = ::crc32(crc, buf.data(), bytesRead);
+            writer.write(buf.data(), bytesRead);
+        }
     }
+    file.close();
 
-    task->write(compressed.data(), compressed.size());
-#endif
+    task->header.crc32 = crc;
+    writer.close();
 
     return Error();
 }