merge: finalize streaming api foundation

Author: LessUp

.github/workflows/ci.yml

@@ -82,14 +82,14 @@ jobs:
           cd algorithms/huffman/go
           go vet ./...
           go test ./...
-          go build -o huffman_go .
+          go build -o huffman_go ./cmd
 
       - name: Build Arithmetic Go
         run: |
           cd algorithms/arithmetic/go
           go vet ./...
           go test ./...
-          go build -o arithmetic_go .
+          go build -o arithmetic_go ./cmd
 
       - name: Build Range coder Go
         run: |
@@ -103,7 +103,7 @@ jobs:
           cd algorithms/rle/go
           go vet ./...
           go test ./...
-          go build -o rle_go .
+          go build -o rle_go ./cmd
 
   # Build and check Rust implementations
   build-rust:
@@ -121,42 +121,39 @@ jobs:
 
       - name: Check Rust formatting
         run: |
-          for file in algorithms/huffman/rust/main.rs algorithms/arithmetic/rust/main.rs algorithms/rle/rust/main.rs; do
-            if [ -f "$file" ]; then
-              rustfmt --check "$file" || exit 1
-            fi
-          done
-          if [ -f "algorithms/range/rust/Cargo.toml" ]; then
-            cd algorithms/range/rust && cargo fmt --check
-          fi
+          cargo fmt --manifest-path algorithms/huffman/rust/Cargo.toml --check
+          cargo fmt --manifest-path algorithms/arithmetic/rust/Cargo.toml --check
+          cargo fmt --manifest-path algorithms/range/rust/Cargo.toml --check
+          cargo fmt --manifest-path algorithms/rle/rust/Cargo.toml --check
+          cargo fmt --manifest-path algorithms/shared/rust/Cargo.toml --check
 
       - name: Build Huffman Rust
         run: |
           cd algorithms/huffman/rust
-          rustc -O main.rs -o huffman_rust
-          rustc --test main.rs -o huffman_rust_test
-          ./huffman_rust_test
+          cargo clippy --all-targets -- -D warnings
+          cargo test
+          cargo build --bin huffman_rust --release
 
       - name: Build Arithmetic Rust
         run: |
           cd algorithms/arithmetic/rust
-          rustc -O main.rs -o arithmetic_rust
-          rustc --test main.rs -o arithmetic_rust_test
-          ./arithmetic_rust_test
+          cargo clippy --all-targets -- -D warnings
+          cargo test
+          cargo build --bin arithmetic_rust --release
 
       - name: Build and check Range coder Rust
         run: |
           cd algorithms/range/rust
-          cargo clippy -- -D warnings
+          cargo clippy --all-targets -- -D warnings
           cargo test
           cargo build --release
 
       - name: Build RLE Rust
         run: |
           cd algorithms/rle/rust
-          rustc -O main.rs -o rle_rust
-          rustc --test main.rs -o rle_rust_test
-          ./rle_rust_test
+          cargo clippy --all-targets -- -D warnings
+          cargo test
+          cargo build --bin rle_rust --release
 
   # Verify encode/decode correctness across languages
   test-correctness:
@@ -200,10 +197,10 @@ jobs:
           (cd algorithms/rle/go && go build -o rle_go ./cmd)
 
           # Rust
-          (cd algorithms/huffman/rust && rustc -O main.rs -o huffman_rust)
-          (cd algorithms/arithmetic/rust && rustc -O main.rs -o arithmetic_rust)
+          (cd algorithms/huffman/rust && cargo build --bin huffman_rust --release)
+          (cd algorithms/arithmetic/rust && cargo build --bin arithmetic_rust --release)
           (cd algorithms/range/rust && cargo build --bin rangecoder --release)
-          (cd algorithms/rle/rust && rustc -O main.rs -o rle_rust)
+          (cd algorithms/rle/rust && cargo build --bin rle_rust --release)
 
       - name: Test Huffman correctness
         run: |
@@ -220,16 +217,16 @@ jobs:
           diff "$TEST_FILE" /tmp/huf_go.dec
 
           echo "Testing Huffman Rust..."
-          ./algorithms/huffman/rust/huffman_rust encode "$TEST_FILE" /tmp/huf_rust.enc
-          ./algorithms/huffman/rust/huffman_rust decode /tmp/huf_rust.enc /tmp/huf_rust.dec
+          ./algorithms/huffman/rust/target/release/huffman_rust encode "$TEST_FILE" /tmp/huf_rust.enc
+          ./algorithms/huffman/rust/target/release/huffman_rust decode /tmp/huf_rust.enc /tmp/huf_rust.dec
           diff "$TEST_FILE" /tmp/huf_rust.dec
 
           echo "Testing cross-language (C++ encode -> Go decode)..."
           ./algorithms/huffman/go/huffman_go decode /tmp/huf_cpp.enc /tmp/huf_cross_go.dec
           diff "$TEST_FILE" /tmp/huf_cross_go.dec
 
           echo "Testing cross-language (C++ encode -> Rust decode)..."
-          ./algorithms/huffman/rust/huffman_rust decode /tmp/huf_cpp.enc /tmp/huf_cross_rust.dec
+          ./algorithms/huffman/rust/target/release/huffman_rust decode /tmp/huf_cpp.enc /tmp/huf_cross_rust.dec
           diff "$TEST_FILE" /tmp/huf_cross_rust.dec
 
           echo "Huffman tests passed!"
@@ -249,16 +246,16 @@ jobs:
           diff "$TEST_FILE" /tmp/arith_go.dec
 
           echo "Testing Arithmetic Rust..."
-          ./algorithms/arithmetic/rust/arithmetic_rust encode "$TEST_FILE" /tmp/arith_rust.enc
-          ./algorithms/arithmetic/rust/arithmetic_rust decode /tmp/arith_rust.enc /tmp/arith_rust.dec
+          ./algorithms/arithmetic/rust/target/release/arithmetic_rust encode "$TEST_FILE" /tmp/arith_rust.enc
+          ./algorithms/arithmetic/rust/target/release/arithmetic_rust decode /tmp/arith_rust.enc /tmp/arith_rust.dec
           diff "$TEST_FILE" /tmp/arith_rust.dec
 
           echo "Testing cross-language (C++ encode -> Go decode)..."
           ./algorithms/arithmetic/go/arithmetic_go decode /tmp/arith_cpp.enc /tmp/arith_cross_go.dec
           diff "$TEST_FILE" /tmp/arith_cross_go.dec
 
           echo "Testing cross-language (C++ encode -> Rust decode)..."
-          ./algorithms/arithmetic/rust/arithmetic_rust decode /tmp/arith_cpp.enc /tmp/arith_cross_rust.dec
+          ./algorithms/arithmetic/rust/target/release/arithmetic_rust decode /tmp/arith_cpp.enc /tmp/arith_cross_rust.dec
           diff "$TEST_FILE" /tmp/arith_cross_rust.dec
 
           echo "Arithmetic tests passed!"
@@ -315,16 +312,16 @@ jobs:
           diff "$TEST_FILE" /tmp/rle_go.dec
 
           echo "Testing RLE Rust..."
-          ./algorithms/rle/rust/rle_rust encode "$TEST_FILE" /tmp/rle_rust.enc
-          ./algorithms/rle/rust/rle_rust decode /tmp/rle_rust.enc /tmp/rle_rust.dec
+          ./algorithms/rle/rust/target/release/rle_rust encode "$TEST_FILE" /tmp/rle_rust.enc
+          ./algorithms/rle/rust/target/release/rle_rust decode /tmp/rle_rust.enc /tmp/rle_rust.dec
           diff "$TEST_FILE" /tmp/rle_rust.dec
 
           echo "Testing cross-language (C++ encode -> Go decode)..."
           ./algorithms/rle/go/rle_go decode /tmp/rle_cpp.enc /tmp/rle_cross_go.dec
           diff "$TEST_FILE" /tmp/rle_cross_go.dec
 
           echo "Testing cross-language (C++ encode -> Rust decode)..."
-          ./algorithms/rle/rust/rle_rust decode /tmp/rle_cpp.enc /tmp/rle_cross_rust.dec
+          ./algorithms/rle/rust/target/release/rle_rust decode /tmp/rle_cpp.enc /tmp/rle_cross_rust.dec
           diff "$TEST_FILE" /tmp/rle_cross_rust.dec
 
           echo "RLE tests passed!"

.gitignore

@@ -46,6 +46,11 @@ coverage.out
 **/huffman_rust_test
 **/arithmetic_rust_test
 **/rle_rust_test
+**/test_lifecycle
+
+# Local environments
+.venv/
+.omc/
 
 # CMake generated files
 CMakeCache.txt

Makefile

@@ -1,5 +1,6 @@
 .PHONY: build build-huffman build-arithmetic build-range build-rle \
        test test-huffman-go test-arithmetic-go test-range-go test-rle-go \
+       test-shared-cpp test-shared-go test-shared-rust \
        test-huffman-rust test-arithmetic-rust test-range-rust test-rle-rust \
        test-data bench clean format lint spec-init spec-list spec-status
 
@@ -8,31 +9,45 @@
 build: build-huffman build-arithmetic build-range build-rle
 
 build-huffman:
-	g++ -std=c++17 -O2 -Wall -Wextra -Werror algorithms/huffman/cpp/main.cpp -o algorithms/huffman/cpp/huffman_cpp
+	g++ -std=c++17 -O2 -Wall -Wextra -Werror -Ialgorithms/shared/cpp/include algorithms/shared/cpp/src/buffer_api.cpp algorithms/huffman/cpp/main.cpp -o algorithms/huffman/cpp/huffman_cpp
 	go build -o algorithms/huffman/go/huffman_go ./algorithms/huffman/go/cmd
-	rustc -O algorithms/huffman/rust/main.rs -o algorithms/huffman/rust/huffman_rust
+	cargo build --manifest-path algorithms/huffman/rust/Cargo.toml --bin huffman_rust --release
+	cp algorithms/huffman/rust/target/release/huffman_rust algorithms/huffman/rust/huffman_rust
 
 build-arithmetic:
-	g++ -std=c++17 -O2 -Wall -Wextra -Werror algorithms/arithmetic/cpp/main.cpp -o algorithms/arithmetic/cpp/arithmetic_cpp
+	g++ -std=c++17 -O2 -Wall -Wextra -Werror -Ialgorithms/shared/cpp/include algorithms/shared/cpp/src/buffer_api.cpp algorithms/arithmetic/cpp/main.cpp -o algorithms/arithmetic/cpp/arithmetic_cpp
 	go build -o algorithms/arithmetic/go/arithmetic_go ./algorithms/arithmetic/go/cmd
-	rustc -O algorithms/arithmetic/rust/main.rs -o algorithms/arithmetic/rust/arithmetic_rust
+	cargo build --manifest-path algorithms/arithmetic/rust/Cargo.toml --bin arithmetic_rust --release
+	cp algorithms/arithmetic/rust/target/release/arithmetic_rust algorithms/arithmetic/rust/arithmetic_rust
 
 build-range:
-	g++ -std=c++17 -O2 -Wall -Wextra -Werror algorithms/range/cpp/main.cpp -o algorithms/range/cpp/rangecoder_cpp
+	g++ -std=c++17 -O2 -Wall -Wextra -Werror -Ialgorithms/shared/cpp/include algorithms/shared/cpp/src/buffer_api.cpp algorithms/range/cpp/main.cpp -o algorithms/range/cpp/rangecoder_cpp
 	go build -o algorithms/range/go/rangecoder_go ./algorithms/range/go/cmd
 	cargo build --manifest-path algorithms/range/rust/Cargo.toml --release
 
 build-rle:
-	g++ -std=c++17 -O2 -Wall -Wextra -Werror algorithms/rle/cpp/main.cpp -o algorithms/rle/cpp/rle_cpp
+	g++ -std=c++17 -O2 -Wall -Wextra -Werror -Ialgorithms/shared/cpp/include algorithms/shared/cpp/src/buffer_api.cpp algorithms/rle/cpp/main.cpp -o algorithms/rle/cpp/rle_cpp
 	go build -o algorithms/rle/go/rle_go ./algorithms/rle/go/cmd
-	rustc -O algorithms/rle/rust/main.rs -o algorithms/rle/rust/rle_rust
+	cargo build --manifest-path algorithms/rle/rust/Cargo.toml --bin rle_rust --release
+	cp algorithms/rle/rust/target/release/rle_rust algorithms/rle/rust/rle_rust
 
 # ── Test ───────────────────────────────────────────────────────────────────
 
 test: test-data \
+      test-shared-cpp test-shared-go test-shared-rust \
       test-huffman-go test-arithmetic-go test-range-go test-rle-go \
       test-huffman-rust test-arithmetic-rust test-range-rust test-rle-rust
 
+test-shared-cpp:
+	g++ -std=c++17 -O2 -Wall -Wextra -Werror -DCOMPRESSKIT_NO_MAIN -Ialgorithms/shared/cpp/include algorithms/shared/cpp/src/buffer_api.cpp algorithms/huffman/cpp/main.cpp algorithms/arithmetic/cpp/main.cpp algorithms/range/cpp/main.cpp algorithms/rle/cpp/main.cpp algorithms/shared/cpp/tests/test_lifecycle.cpp -o algorithms/shared/cpp/tests/test_lifecycle
+	./algorithms/shared/cpp/tests/test_lifecycle
+
+test-shared-go:
+	go test ./algorithms/shared/go/...
+
+test-shared-rust:
+	cargo test --manifest-path algorithms/shared/rust/Cargo.toml
+
 test-huffman-go:
 	go test ./algorithms/huffman/go/... ./algorithms/huffman/go/cmd/...
 
@@ -46,19 +61,16 @@ test-rle-go:
 	go test ./algorithms/rle/go/... ./algorithms/rle/go/cmd/...
 
 test-huffman-rust:
-	rustc --test algorithms/huffman/rust/main.rs -o algorithms/huffman/rust/huffman_rust_test
-	./algorithms/huffman/rust/huffman_rust_test
+	cargo test --manifest-path algorithms/huffman/rust/Cargo.toml
 
 test-arithmetic-rust:
-	rustc --test algorithms/arithmetic/rust/main.rs -o algorithms/arithmetic/rust/arithmetic_rust_test
-	./algorithms/arithmetic/rust/arithmetic_rust_test
+	cargo test --manifest-path algorithms/arithmetic/rust/Cargo.toml
 
 test-range-rust:
 	cargo test --manifest-path algorithms/range/rust/Cargo.toml
 
 test-rle-rust:
-	rustc --test algorithms/rle/rust/main.rs -o algorithms/rle/rust/rle_rust_test
-	./algorithms/rle/rust/rle_rust_test
+	cargo test --manifest-path algorithms/rle/rust/Cargo.toml
 
 # ── Data / Bench / Clean ──────────────────────────────────────────────────
 
@@ -83,19 +95,25 @@ format:
 	@echo "Formatting Go code..."
 	gofmt -w algorithms/*/go
 	@echo "Formatting Rust code..."
-	rustfmt algorithms/huffman/rust/main.rs algorithms/arithmetic/rust/main.rs algorithms/rle/rust/main.rs 2>/dev/null || true
+	cargo fmt --manifest-path algorithms/huffman/rust/Cargo.toml 2>/dev/null || true
+	cargo fmt --manifest-path algorithms/arithmetic/rust/Cargo.toml 2>/dev/null || true
+	cargo fmt --manifest-path algorithms/rle/rust/Cargo.toml 2>/dev/null || true
 	cd algorithms/range/rust && cargo fmt 2>/dev/null || true
 	@echo "Formatting C++ code (if clang-format available)..."
-	@for f in algorithms/*/cpp/main.cpp; do \
+	@for f in algorithms/*/cpp/main.cpp algorithms/shared/cpp/src/buffer_api.cpp algorithms/shared/cpp/tests/test_lifecycle.cpp; do \
 		clang-format -i "$$f" 2>/dev/null || true; \
 	done
 	@echo "Done!"
 
 lint:
 	@echo "Linting Go code..."
-	go vet ./algorithms/huffman/go/... ./algorithms/arithmetic/go/... ./algorithms/range/go/... ./algorithms/rle/go/... 2>/dev/null || true
+	go vet ./algorithms/shared/go/... ./algorithms/huffman/go/... ./algorithms/arithmetic/go/... ./algorithms/range/go/... ./algorithms/rle/go/... 2>/dev/null || true
 	@echo "Linting Rust code..."
+	cd algorithms/huffman/rust && cargo clippy --all-targets -- -D warnings 2>/dev/null || true
+	cd algorithms/arithmetic/rust && cargo clippy --all-targets -- -D warnings 2>/dev/null || true
 	cd algorithms/range/rust && cargo clippy -- -D warnings 2>/dev/null || true
+	cd algorithms/rle/rust && cargo clippy --all-targets -- -D warnings 2>/dev/null || true
+	cd algorithms/shared/rust && cargo clippy --all-targets -- -D warnings 2>/dev/null || true
 	@echo "Done!"
 
 # ── OpenSpec ────────────────────────────────────────────────────────────────

algorithms/arithmetic/cpp/main.cpp

@@ -4,6 +4,8 @@
 #include <iostream>
 #include <string>
 
+#include "compresskit/buffer_api.hpp"
+
 class BitWriter {
 public:
     explicit BitWriter(std::ostream& s) : stream(s), buffer(0), bits_in_buffer(0) {}
@@ -391,6 +393,15 @@ static bool decompress_file(const std::string& input_path, const std::string& ou
     return true;
 }
 
+bool arithmetic_encode_file(const std::string& input_path, const std::string& output_path) {
+    return compress_file(input_path, output_path);
+}
+
+bool arithmetic_decode_file(const std::string& input_path, const std::string& output_path) {
+    return decompress_file(input_path, output_path);
+}
+
+#ifndef COMPRESSKIT_NO_MAIN
 int main(int argc, char** argv) {
     if (argc != 4) {
         std::cerr << "Usage: " << argv[0] << " encode|decode input output\n";
@@ -403,13 +414,14 @@ int main(int argc, char** argv) {
     bool ok = true;
 
     if (mode == "encode") {
-        ok = compress_file(input_path, output_path);
+        ok = compresskit::encode_file_via_buffer(arithmetic_encode_file, input_path, output_path);
     } else if (mode == "decode") {
-        ok = decompress_file(input_path, output_path);
+        ok = compresskit::decode_file_via_buffer(arithmetic_decode_file, input_path, output_path);
     } else {
         std::cerr << "Unknown mode\n";
         return 1;
     }
 
     return ok ? 0 : 1;
 }
+#endif

algorithms/arithmetic/go/arithmetic.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"io"
 	"os"
+
+	"github.com/LessUp/compress-kit/algorithms/shared/go/codec"
 )
 
 const (
@@ -415,12 +417,17 @@ func Decode(r io.Reader, w io.Writer) error {
 	bw := bufio.NewWriter(w)
 	bitReader := NewBitReader(br)
 	decoder := NewArithmeticDecoder(bitReader)
+	var totalWritten uint64
 
 	for {
 		sym := decoder.DecodeSymbol(cumulative)
 		if sym == uint32(EOFSymbol) {
 			break
 		}
+		totalWritten++
+		if totalWritten > codec.MaxOutputSize {
+			return fmt.Errorf("output size limit exceeded")
+		}
 		if err := bw.WriteByte(byte(sym)); err != nil {
 			return err
 		}
@@ -431,34 +438,38 @@ func Decode(r io.Reader, w io.Writer) error {
 
 // EncodeFile is a convenience function for file-based encoding.
 func EncodeFile(inputPath, outputPath string) error {
-	in, err := os.Open(inputPath)
+	input, err := os.ReadFile(inputPath)
 	if err != nil {
 		return fmt.Errorf("cannot open input file: %s: %w", inputPath, err)
 	}
-	defer in.Close()
 
-	out, err := os.Create(outputPath)
+	encoded, err := codec.EncodeBuffer(NewStreamingEncoder(), input)
 	if err != nil {
+		return err
+	}
+
+	if err := os.WriteFile(outputPath, encoded, 0o644); err != nil {
 		return fmt.Errorf("cannot open output file: %s: %w", outputPath, err)
 	}
-	defer out.Close()
 
-	return Encode(in, out)
+	return nil
 }
 
 // DecodeFile is a convenience function for file-based decoding.
 func DecodeFile(inputPath, outputPath string) error {
-	in, err := os.Open(inputPath)
+	input, err := os.ReadFile(inputPath)
 	if err != nil {
 		return fmt.Errorf("cannot open input file: %s: %w", inputPath, err)
 	}
-	defer in.Close()
 
-	out, err := os.Create(outputPath)
+	decoded, err := codec.DecodeBuffer(NewStreamingDecoder(), input)
 	if err != nil {
+		return err
+	}
+
+	if err := os.WriteFile(outputPath, decoded, 0o644); err != nil {
 		return fmt.Errorf("cannot open output file: %s: %w", outputPath, err)
 	}
-	defer out.Close()
 
-	return Decode(in, out)
+	return nil
 }