Skip to content

Commit 57fc9e5

Browse files
LessUpqwencoder
LessUp
and
qwencoder
committed
chore: update encoding implementations across languages
- Update arithmetic, Huffman, range, and RLE encoding implementations - Improve C++, Go, and Rust versions - Update Makefile for better build automation Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
1 parent 225cbc3 commit 57fc9e5

13 files changed

Lines changed: 321 additions & 203 deletions

File tree

‎Makefile‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,22 @@
88
build: build-huffman build-arithmetic build-range build-rle
99

1010
build-huffman:
11-
g++ -std=c++17 -O2 huffman/cpp/main.cpp -o huffman/cpp/huffman_cpp
11+
g++ -std=c++17 -O2 -Wall -Wextra -Werror huffman/cpp/main.cpp -o huffman/cpp/huffman_cpp
1212
go build -o huffman/go/huffman_go ./huffman/go
1313
rustc -O huffman/rust/main.rs -o huffman/rust/huffman_rust
1414

1515
build-arithmetic:
16-
g++ -std=c++17 -O2 arithmetic/cpp/main.cpp -o arithmetic/cpp/arithmetic_cpp
16+
g++ -std=c++17 -O2 -Wall -Wextra -Werror arithmetic/cpp/main.cpp -o arithmetic/cpp/arithmetic_cpp
1717
go build -o arithmetic/go/arithmetic_go ./arithmetic/go
1818
rustc -O arithmetic/rust/main.rs -o arithmetic/rust/arithmetic_rust
1919

2020
build-range:
21-
g++ -std=c++17 -O2 range/cpp/main.cpp -o range/cpp/rangecoder_cpp
21+
g++ -std=c++17 -O2 -Wall -Wextra -Werror range/cpp/main.cpp -o range/cpp/rangecoder_cpp
2222
go build -o range/go/rangecoder_go ./range/go/cmd
2323
cargo build --manifest-path range/rust/Cargo.toml --release
2424

2525
build-rle:
26-
g++ -std=c++17 -O2 rle/cpp/main.cpp -o rle/cpp/rle_cpp
26+
g++ -std=c++17 -O2 -Wall -Wextra -Werror rle/cpp/main.cpp -o rle/cpp/rle_cpp
2727
go build -o rle/go/rle_go ./rle/go
2828
rustc -O rle/rust/main.rs -o rle/rust/rle_rust
2929

‎arithmetic/cpp/main.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ class ArithmeticDecoder {
197197
static const uint32_t SYMBOL_LIMIT = 257;
198198
static const uint32_t EOF_SYMBOL = SYMBOL_LIMIT - 1;
199199
static const uint32_t MAX_TOTAL = 1u << 24;
200+
static const uint64_t MAX_INPUT_SIZE = 4ULL * 1024 * 1024 * 1024; // 4 GiB max
200201

201202
static void scale_frequencies(std::vector<uint32_t>& freq) {
202203
uint64_t total = 0;
@@ -293,6 +294,18 @@ static bool read_frequencies(std::istream& in, std::vector<uint32_t>& freq) {
293294
}
294295

295296
static bool compress_file(const std::string& input_path, const std::string& output_path) {
297+
// Check input file size to prevent frequency overflow
298+
{
299+
std::ifstream check(input_path, std::ios::binary | std::ios::ate);
300+
if (check) {
301+
auto size = check.tellg();
302+
if (size > 0 && static_cast<uint64_t>(size) > MAX_INPUT_SIZE) {
303+
std::cerr << "Input file too large (max " << MAX_INPUT_SIZE << " bytes)\n";
304+
return false;
305+
}
306+
}
307+
}
308+
296309
std::vector<uint32_t> freq = build_frequencies_from_file(input_path);
297310
std::vector<uint32_t> cumulative = build_cumulative(freq);
298311

‎arithmetic/go/main.go‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,17 @@ import (
88
"os"
99
)
1010

11-
// 算术编码 Go 实现。
12-
// 文件格式与 C++ 实现完全一致，支持交叉编解码验证。
11+
// Arithmetic coding Go implementation.
12+
// File format is fully compatible with C++ implementation, supports cross-language encode/decode verification.
1313
// Magic: AENC (4 bytes)
14-
// 频率表: count(4 bytes LE) + count × freq(4 bytes LE)
15-
// 算术编码比特流
14+
// Frequency table: count(4 bytes LE) + count × freq(4 bytes LE)
15+
// Arithmetic coding bitstream
1616

1717
const (
18-
SymbolLimit = 257
19-
EOFSymbol = SymbolLimit - 1
20-
MaxTotal = uint32(1) << 24
18+
SymbolLimit = 257
19+
EOFSymbol = SymbolLimit - 1
20+
MaxTotal = uint32(1) << 24
21+
MaxInputSize = 4 * 1024 * 1024 * 1024 // 4 GiB max to prevent frequency overflow
2122

2223
stateBits = 32
2324
fullRange = uint64(1) << stateBits
@@ -240,7 +241,7 @@ func (d *ArithmeticDecoder) DecodeSymbol(cumulative []uint32) uint32 {
240241
}
241242

242243
// ---------------------------------------------------------------------------
243-
// 频率表处理
244+
// Frequency table processing
244245
// ---------------------------------------------------------------------------
245246

246247
func scaleFrequencies(freq []uint32) {
@@ -284,9 +285,19 @@ func buildFrequenciesFromFile(path string) ([]uint32, error) {
284285
freq := make([]uint32, SymbolLimit)
285286
f, err := os.Open(path)
286287
if err != nil {
287-
return nil, fmt.Errorf("无法打开输入文件用于读取: %s: %w", path, err)
288+
return nil, fmt.Errorf("cannot open input file for reading: %s: %w", path, err)
288289
}
289290
defer f.Close()
291+
292+
// Check file size to prevent frequency overflow
293+
stat, err := f.Stat()
294+
if err != nil {
295+
return nil, fmt.Errorf("cannot stat input file: %w", err)
296+
}
297+
if stat.Size() > MaxInputSize {
298+
return nil, fmt.Errorf("input file too large (max %d bytes)", MaxInputSize)
299+
}
300+
290301
r := bufio.NewReader(f)
291302
for {
292303
b, err := r.ReadByte()
@@ -329,20 +340,20 @@ func writeFrequencies(w io.Writer, freq []uint32) error {
329340
func readFrequencies(r io.Reader) ([]uint32, error) {
330341
var count uint32
331342
if err := binary.Read(r, binary.LittleEndian, &count); err != nil {
332-
return nil, fmt.Errorf("读取频率表失败: %w", err)
343+
return nil, fmt.Errorf("failed to read frequency table: %w", err)
333344
}
334345
if count != uint32(SymbolLimit) {
335-
return nil, fmt.Errorf("频率表大小异常: %d", count)
346+
return nil, fmt.Errorf("invalid frequency table size: %d", count)
336347
}
337348
freq := make([]uint32, count)
338349
if err := binary.Read(r, binary.LittleEndian, freq); err != nil {
339-
return nil, fmt.Errorf("读取频率表失败: %w", err)
350+
return nil, fmt.Errorf("failed to read frequency table: %w", err)
340351
}
341352
return freq, nil
342353
}
343354

344355
// ---------------------------------------------------------------------------
345-
// 压缩 / 解压
356+
// Compression / Decompression
346357
// ---------------------------------------------------------------------------
347358

348359
func compressFile(inputPath, outputPath string) error {
@@ -354,13 +365,13 @@ func compressFile(inputPath, outputPath string) error {
354365

355366
inFile, err := os.Open(inputPath)
356367
if err != nil {
357-
return fmt.Errorf("无法打开输入文件用于读取: %s: %w", inputPath, err)
368+
return fmt.Errorf("cannot open input file for reading: %s: %w", inputPath, err)
358369
}
359370
defer inFile.Close()
360371

361372
outFile, err := os.Create(outputPath)
362373
if err != nil {
363-
return fmt.Errorf("无法打开输出文件用于写入: %s: %w", outputPath, err)
374+
return fmt.Errorf("cannot open output file for writing: %s: %w", outputPath, err)
364375
}
365376
defer outFile.Close()
366377

@@ -381,7 +392,7 @@ func compressFile(inputPath, outputPath string) error {
381392
break
382393
}
383394
if err != nil {
384-
return fmt.Errorf("读取输入文件失败: %w", err)
395+
return fmt.Errorf("failed to read input file: %w", err)
385396
}
386397
if err := encoder.EncodeSymbol(uint32(b), cumulative); err != nil {
387398
return err
@@ -396,14 +407,14 @@ func compressFile(inputPath, outputPath string) error {
396407
func decompressFile(inputPath, outputPath string) error {
397408
inFile, err := os.Open(inputPath)
398409
if err != nil {
399-
return fmt.Errorf("无法打开输入文件用于读取: %s: %w", inputPath, err)
410+
return fmt.Errorf("cannot open input file for reading: %s: %w", inputPath, err)
400411
}
401412
defer inFile.Close()
402413
r := bufio.NewReader(inFile)
403414

404415
magic := make([]byte, 4)
405416
if _, err := io.ReadFull(r, magic); err != nil || magic[0] != 'A' || magic[1] != 'E' || magic[2] != 'N' || magic[3] != 'C' {
406-
return fmt.Errorf("输入文件格式非法")
417+
return fmt.Errorf("invalid input file format")
407418
}
408419

409420
freq, err := readFrequencies(r)
@@ -414,7 +425,7 @@ func decompressFile(inputPath, outputPath string) error {
414425

415426
outFile, err := os.Create(outputPath)
416427
if err != nil {
417-
return fmt.Errorf("无法打开输出文件用于写入: %s: %w", outputPath, err)
428+
return fmt.Errorf("cannot open output file for writing: %s: %w", outputPath, err)
418429
}
419430
defer outFile.Close()
420431
w := bufio.NewWriter(outFile)
@@ -451,7 +462,7 @@ func main() {
451462
} else if mode == "decode" {
452463
err = decompressFile(inputPath, outputPath)
453464
} else {
454-
fmt.Fprintln(os.Stderr, "未知模式，应为 encode 或 decode")
465+
fmt.Fprintln(os.Stderr, "unknown mode, expected encode or decode")
455466
os.Exit(1)
456467
}
457468
if err != nil {

‎arithmetic/rust/main.rs‎

Lines changed: 26 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@ use std::fs::File;
33
use std::io::{self, BufReader, BufWriter, Read, Write};
44
use std::process;
55

6-
// 算术编码 Rust 实现。
7-
// 文件格式与 C++/Go 实现完全一致，支持交叉编解码验证。
6+
// Arithmetic coding Rust implementation.
7+
// File format is fully compatible with C++/Go implementations, supports cross-language encode/decode verification.
88
// Magic: AENC (4 bytes)
9-
// 频率表: count(4 bytes LE) + count × freq(4 bytes LE)
10-
// 算术编码比特流
9+
// Frequency table: count(4 bytes LE) + count × freq(4 bytes LE)
10+
// Arithmetic coding bitstream
1111

1212
const SYMBOL_LIMIT: usize = 257;
1313
const EOF_SYMBOL: u32 = (SYMBOL_LIMIT - 1) as u32;
1414
const MAX_TOTAL: u32 = 1 << 24;
15+
const MAX_INPUT_SIZE: u64 = 4 * 1024 * 1024 * 1024; // 4 GiB max
1516

1617
const STATE_BITS: u64 = 32;
1718
const FULL_RANGE: u64 = 1u64 << STATE_BITS;
@@ -240,7 +241,7 @@ impl<R: Read> ArithmeticDecoder<R> {
240241
}
241242

242243
// ---------------------------------------------------------------------------
243-
// 频率表处理
244+
// Frequency table processing
244245
// ---------------------------------------------------------------------------
245246

246247
fn scale_frequencies(freq: &mut [u32]) {
@@ -280,7 +281,17 @@ fn scale_frequencies(freq: &mut [u32]) {
280281
fn build_frequencies_from_file(path: &str) -> io::Result<Vec<u32>> {
281282
let mut freq = vec![0u32; SYMBOL_LIMIT];
282283
let file = File::open(path)
283-
.map_err(|e| io::Error::new(e.kind(), format!("无法打开输入文件用于读取: {path}: {e}")))?;
284+
.map_err(|e| io::Error::new(e.kind(), format!("cannot open input file for reading: {path}: {e}")))?;
285+
286+
// Check file size to prevent frequency overflow
287+
let metadata = file.metadata()?;
288+
if metadata.len() > MAX_INPUT_SIZE {
289+
return Err(io::Error::new(
290+
io::ErrorKind::Other,
291+
format!("input file too large (max {} bytes)", MAX_INPUT_SIZE),
292+
));
293+
}
294+
284295
let mut reader = BufReader::new(file);
285296
let mut buf = [0u8; 4096];
286297
loop {
@@ -291,7 +302,7 @@ fn build_frequencies_from_file(path: &str) -> io::Result<Vec<u32>> {
291302
freq[b as usize] += 1;
292303
}
293304
}
294-
Err(_) => break,
305+
Err(e) => return Err(e),
295306
}
296307
}
297308
freq[EOF_SYMBOL as usize] = 1;
@@ -327,27 +338,27 @@ fn read_frequencies<R: Read>(reader: &mut R) -> io::Result<Vec<u32>> {
327338
let mut count_bytes = [0u8; 4];
328339
reader
329340
.read_exact(&mut count_bytes)
330-
.map_err(|e| io::Error::new(e.kind(), format!("读取频率表失败: {e}")))?;
341+
.map_err(|e| io::Error::new(e.kind(), format!("failed to read frequency table: {e}")))?;
331342
let count = u32::from_le_bytes(count_bytes) as usize;
332343
if count != SYMBOL_LIMIT {
333344
return Err(io::Error::new(
334345
io::ErrorKind::InvalidData,
335-
format!("频率表大小异常: {count}"),
346+
format!("invalid frequency table size: {count}"),
336347
));
337348
}
338349
let mut freq = vec![0u32; count];
339350
for f in freq.iter_mut() {
340351
let mut arr = [0u8; 4];
341352
reader
342353
.read_exact(&mut arr)
343-
.map_err(|e| io::Error::new(e.kind(), format!("读取频率表失败: {e}")))?;
354+
.map_err(|e| io::Error::new(e.kind(), format!("failed to read frequency table: {e}")))?;
344355
*f = u32::from_le_bytes(arr);
345356
}
346357
Ok(freq)
347358
}
348359

349360
// ---------------------------------------------------------------------------
350-
// 压缩 / 解压
361+
// Compression / Decompression
351362
// ---------------------------------------------------------------------------
352363

353364
fn compress_file(input_path: &str, output_path: &str) -> io::Result<()> {
@@ -388,7 +399,7 @@ fn decompress_file(input_path: &str, output_path: &str) -> io::Result<()> {
388399
if &magic != b"AENC" {
389400
return Err(io::Error::new(
390401
io::ErrorKind::InvalidData,
391-
"输入文件格式非法",
402+
"invalid input file format",
392403
));
393404
}
394405
let freq = read_frequencies(&mut reader)?;
@@ -505,7 +516,7 @@ mod tests {
505516
fn main() {
506517
let args: Vec<String> = env::args().collect();
507518
if args.len() != 4 {
508-
eprintln!("用法: {} encode|decode input output", args[0]);
519+
eprintln!("usage: {} encode|decode input output", args[0]);
509520
process::exit(1);
510521
}
511522
let mode = &args[1];
@@ -516,13 +527,13 @@ fn main() {
516527
"encode" => arithmetic_encode_file(input_path, output_path),
517528
"decode" => arithmetic_decode_file(input_path, output_path),
518529
_ => {
519-
eprintln!("未知模式，应为 encode 或 decode");
530+
eprintln!("unknown mode, expected encode or decode");
520531
process::exit(1);
521532
}
522533
};
523534

524535
if let Err(e) = result {
525-
eprintln!("运行失败: {e}");
536+
eprintln!("execution failed: {e}");
526537
process::exit(1);
527538
}
528539
}

0 commit comments

Comments
 (0)