[Feat] Complete process_trace and reader_init_param (#271)

* [FEAT] Complete process_trace and reader_init_param

* Apply copilot suggestion

* Add check for zero division

* Cleanup

* Feat: unify process_trace output

* Ruff check

* Add ruff lint and format check into pre-commit

* Fix copilot comments

Author: haochengxia

README.md

@@ -333,8 +333,8 @@ import libcachesim as lcs
 reader = lcs.create_zipf_requests(num_objects=1000, num_requests=10000) # synthetic trace
 # reader = lcs.open_trace("./data/cloudPhysicsIO.oracleGeneral.bin") # real trace
 cache = lcs.FIFO(cache_size=1024*1024)
-miss_ratio = cache.process_trace(reader)
-print(f"Miss ratio: {miss_ratio:.4f}")
+obj_miss_ratio, byte_miss_ratio = cache.process_trace(reader)
+print(f"Obj miss ratio: {obj_miss_ratio:.4f}, byte miss ratio: {byte_miss_ratio:.4f}")
 ```
 
 ### Extending new algorithm
@@ -344,6 +344,7 @@ With python package, you can extend new algorithm to test your own eviction desi
 ```python
 import libcachesim as lcs
 from collections import deque
+from contextlib import suppress
 
 cache = lcs.PythonHookCachePolicy(cache_size=1024, cache_name="CustomFIFO")
 
@@ -360,15 +361,18 @@ def eviction_hook(fifo_queue, obj_id, obj_size):
     return fifo_queue[0]  # Return first item (oldest)
 
 def remove_hook(fifo_queue, obj_id):
-    if fifo_queue and fifo_queue[0] == obj_id:
-        fifo_queue.popleft()
+    with suppress(ValueError):
+        fifo_queue.remove(obj_id)
 
 # Set the hooks and test
 cache.set_hooks(init_hook, hit_hook, miss_hook, eviction_hook, remove_hook)
 
-reader = lcs.open_trace("./data/cloudPhysicsIO.oracleGeneral.bin")
-miss_ratio = cache.process_trace(reader)
-print(f"Miss ratio: {miss_ratio:.4f}")
+reader = lcs.open_trace(
+    trace_path="./data/cloudPhysicsIO.oracleGeneral.bin",
+    params=lcs.ReaderInitParam(ignore_obj_size=True)
+)
+obj_miss_ratio, byte_miss_ratio = cache.process_trace(reader)
+print(f"Obj miss ratio: {obj_miss_ratio:.4f}, byte miss ratio: {byte_miss_ratio:.4f}")
 ```
 
 

doc/quickstart_plugin.md

@@ -264,8 +264,8 @@ cache.set_hooks(init_hook, hit_hook, miss_hook, eviction_hook, remove_hook)
 
 # Process traces efficiently
 reader = lcs.open_trace("./data/cloudPhysicsIO.vscsi", lcs.TraceType.VSCSI_TRACE)
-miss_ratio = cache.process_trace(reader)
-print(f"Miss ratio: {miss_ratio:.4f}")
+obj_miss_ratio, byte_miss_ratio = cache.process_trace(reader)
+print(f"Obj miss ratio: {obj_miss_ratio:.4f}, byte miss ratio: {byte_miss_ratio:.4f}")
 
 # Or process individual requests
 req = lcs.Request(obj_id=1, obj_size=100)

libCacheSim-python/README.md

@@ -3,8 +3,7 @@
 [![Python Release](https://github.com/1a1a11a/libCacheSim/actions/workflows/pypi-release.yml/badge.svg)](https://github.com/1a1a11a/libCacheSim/actions/workflows/pypi-release.yml)
 [![Python Versions](https://img.shields.io/pypi/pyversions/libcachesim.svg?logo=python&logoColor=white)](https://pypi.org/project/libcachesim)
 [![PyPI Version](https://img.shields.io/pypi/v/libcachesim.svg?)](https://pypi.org/project/libcachesim)
-![PyPI - Downloads](https://img.shields.io/pypi/dd/libcachesim)
-
+[![PyPI - Downloads](https://img.shields.io/pypi/dd/libcachesim)](https://pypistats.org/packages/libcachesim)
 
 Python bindings for libCacheSim, a high-performance cache simulator and analysis library.
 
@@ -63,24 +62,35 @@ print(cache.get(req))  # True (second access)
 
 ### Trace Processing
 
+To simulate with traces, we need to read the request of traces correctly. `open_trace` is an unified interface for trace reading, which accepet three parameters:
+
+- `trace_path`: trace path, can be relative or absolutive path.
+- `type` (optional): if not given, we will automatically infer the type of trace according to the suffix of the trace file.
+- `params` (optional): if not given, default params are applied.
+
 ```python
 import libcachesim as lcs
 
 # Open trace and process efficiently
-reader = lcs.open_trace("./data/cloudPhysicsIO.oracleGeneral.bin", lcs.TraceType.ORACLE_GENERAL_TRACE)
+reader = lcs.open_trace(
+    trace_path = "./data/cloudPhysicsIO.oracleGeneral.bin",
+    type = lcs.TraceType.ORACLE_GENERAL_TRACE,
+    params = lcs.ReaderInitParam(ignore_obj_size=True)
+)
 cache = lcs.S3FIFO(cache_size=1024*1024)
 
 # Process entire trace efficiently (C++ backend)
-miss_ratio = cache.process_trace(reader)
-print(f"Miss ratio: {miss_ratio:.4f}")
+obj_miss_ratio, byte_miss_ratio = cache.process_trace(reader)
+print(f"Object miss ratio: {obj_miss_ratio:.4f}, Byte miss ratio: {byte_miss_ratio:.4f}")
 
+cache = lcs.S3FIFO(cache_size=1024*1024)
 # Process with limits and time ranges
-miss_ratio = cache.process_trace(
+obj_miss_ratio, byte_miss_ratio = cache.process_trace(
     reader,
-    start_req=100,
+    start_req=0,
     max_req=1000
 )
-print(f"Miss ratio: {miss_ratio:.4f}")
+print(f"Object miss ratio: {obj_miss_ratio:.4f}, Byte miss ratio: {byte_miss_ratio:.4f}")
 ```
 
 ## Custom Cache Policies
@@ -147,8 +157,8 @@ print(f"Cache hit: {hit}")  # Should be False (miss)
 ```python
 import libcachesim as lcs
 from collections import deque
+from contextlib import suppress
 
-# Create a custom FIFO cache
 cache = lcs.PythonHookCachePolicy(cache_size=1024, cache_name="CustomFIFO")
 
 def init_hook(cache_size):
@@ -164,15 +174,13 @@ def eviction_hook(fifo_queue, obj_id, obj_size):
     return fifo_queue[0]  # Return first item (oldest)
 
 def remove_hook(fifo_queue, obj_id):
-    if fifo_queue and fifo_queue[0] == obj_id:
-        fifo_queue.popleft()
+    with suppress(ValueError):
+        fifo_queue.remove(obj_id)
 
 # Set the hooks and test
 cache.set_hooks(init_hook, hit_hook, miss_hook, eviction_hook, remove_hook)
 
-req = lcs.Request()
-req.obj_id = 1
-req.obj_size = 100
+req = lcs.Request(obj_id=1, obj_size=100)
 hit = cache.get(req)
 print(f"Cache hit: {hit}")  # Should be False (miss)
 ```
@@ -225,6 +233,7 @@ req = lcs.Request()
 req.obj_id = 1
 req.obj_size = 100
 hit = lru_cache.get(req)
+print(hit)
 ```
 
 ## Examples and Testing
@@ -238,8 +247,8 @@ def compare_algorithms(trace_path):
     algorithms = ['LRU', 'S3FIFO', 'Sieve', 'ARC']
     for algo_name in algorithms:
         cache = getattr(lcs, algo_name)(cache_size=1024*1024)
-        miss_ratio = cache.process_trace(reader)
-        print(f"{algo_name}\t\t{miss_ratio:.4f}")
+        obj_miss_ratio, byte_miss_ratio = cache.process_trace(reader)
+        print(f"{algo_name}\t\tObj: {obj_miss_ratio:.4f}, Byte: {byte_miss_ratio:.4f}")
 
 compare_algorithms("./data/cloudPhysicsIO.vscsi")
 ```
@@ -299,8 +308,8 @@ caches = [
 ]
 
 for i, cache in enumerate(caches):
-    miss_ratio_oracle = cache.process_trace(oracle_reader)
-    miss_ratio_csv = cache.process_trace(csv_reader)
+    miss_ratio_oracle = cache.process_trace(oracle_reader)[0]
+    miss_ratio_csv = cache.process_trace(csv_reader)[0]
     print(f"Cache {i} miss ratio: {miss_ratio_oracle:.4f}, {miss_ratio_csv:.4f}")
 ```
 

libCacheSim-python/examples/README.md

@@ -257,7 +257,7 @@ python -m pytest ../tests/ -v
 3. **Process large traces with C++ backend**:
    ```python
    # Fast: C++ processing
-   miss_ratio = lcs.process_trace(cache, reader)
+   obj_miss_ratio, byte_miss_ratio = lcs.process_trace(cache, reader)
 
    # Slow: Python loop
    for req in reader:

libCacheSim-python/examples/demo_unified_interface.py

@@ -9,7 +9,7 @@
 import os
 
 # Add parent directory for development testing
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
 try:
     import libcachesim as lcs
@@ -24,9 +24,7 @@
 def create_trace_reader():
     """Helper function to create a trace reader."""
     data_file = os.path.join(
-        os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
-        "data",
-        "cloudPhysicsIO.oracleGeneral.bin"
+        os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data", "cloudPhysicsIO.oracleGeneral.bin"
     )
     if not os.path.exists(data_file):
         print(f"Warning: Trace file not found at {data_file}")
@@ -98,8 +96,10 @@ def demo_unified_interface():
     print("   All caches provide: cache_size, n_obj, occupied_byte, n_req")
 
     for name, cache in caches.items():
-        print(f"   {name:20s}: size={cache.cache_size}, objs={cache.n_obj}, "
-              f"bytes={cache.occupied_byte}, reqs={cache.n_req}")
+        print(
+            f"   {name:20s}: size={cache.cache_size}, objs={cache.n_obj}, "
+            f"bytes={cache.occupied_byte}, reqs={cache.n_req}"
+        )
 
     # Demo 3: Efficient trace processing
     print("\n3. Efficient Trace Processing Interface:")
@@ -114,8 +114,8 @@ def demo_unified_interface():
             print(f"   {name:20s}: trace file not available")
             continue
 
-        miss_ratio = cache.process_trace(reader, max_req=max_requests)
-        print(f"   {name:20s}: miss_ratio={miss_ratio:.4f}")
+        obj_miss_ratio, byte_miss_ratio = cache.process_trace(reader, max_req=max_requests)
+        print(f"   {name:20s}: obj_miss_ratio={obj_miss_ratio:.4f}, byte_miss_ratio={byte_miss_ratio:.4f}")
 
     print("\nKey Benefits of Unified Interface:")
     print("   • Same API for all cache policies (built-in + custom)")

libCacheSim-python/examples/python_hook_cache_example.py

@@ -126,7 +126,11 @@ def test_cache_policy(cache, name):
 
     # Test requests: obj_id, obj_size
     test_requests = [
-        (1, 100), (2, 100), (3, 100), (4, 100), (5, 100),  # Fill cache
+        (1, 100),
+        (2, 100),
+        (3, 100),
+        (4, 100),
+        (5, 100),  # Fill cache
         (1, 100),  # Hit
         (6, 100),  # Miss, should evict something
         (2, 100),  # Hit or miss depending on policy

libCacheSim-python/examples/stream_request_example.py

@@ -32,9 +32,9 @@ def main():
     zipf_generator = lcs.create_zipf_requests(
         num_objects=num_objects,
         num_requests=num_requests,
-        alpha=1.0,             # Zipf skewness
-        obj_size=obj_size,     # Object size in bytes
-        seed=42                # For reproducibility
+        alpha=1.0,  # Zipf skewness
+        obj_size=obj_size,  # Object size in bytes
+        seed=42,  # For reproducibility
     )
 
     print(f"Cache size: {cache_size // 1024 // 1024}MB")
@@ -61,10 +61,7 @@ def main():
 
     # Create a uniform-distributed request generator
     uniform_generator = lcs.create_uniform_requests(
-        num_objects=num_objects,
-        num_requests=num_requests,
-        obj_size=obj_size,
-        seed=42
+        num_objects=num_objects, num_requests=num_requests, obj_size=obj_size, seed=42
     )
 
     print(f"Generated {num_requests} uniform requests for {num_objects} objects")
@@ -94,11 +91,7 @@ def main():
 
     for alpha in alphas:
         generator = lcs.create_zipf_requests(
-            num_objects=num_objects,
-            num_requests=num_requests,
-            alpha=alpha,
-            obj_size=obj_size,
-            seed=42
+            num_objects=num_objects, num_requests=num_requests, alpha=alpha, obj_size=obj_size, seed=42
         )
 
         cache = lcs.LRU(cache_size=cache_size)
@@ -123,16 +116,12 @@ def main():
 
     # Fixed workload
     generator = lcs.create_zipf_requests(
-        num_objects=num_objects,
-        num_requests=num_requests,
-        alpha=1.0,
-        obj_size=obj_size,
-        seed=42
+        num_objects=num_objects, num_requests=num_requests, alpha=1.0, obj_size=obj_size, seed=42
     )
 
     cache_sizes = [
-        1 * 1024 * 1024,   # 1MB
-        5 * 1024 * 1024,   # 5MB
+        1 * 1024 * 1024,  # 1MB
+        5 * 1024 * 1024,  # 5MB
         10 * 1024 * 1024,  # 10MB
         50 * 1024 * 1024,  # 50MB
     ]
@@ -145,18 +134,14 @@ def main():
 
         # Create fresh generator for each test
         test_generator = lcs.create_zipf_requests(
-            num_objects=num_objects,
-            num_requests=num_requests,
-            alpha=1.0,
-            obj_size=obj_size,
-            seed=42
+            num_objects=num_objects, num_requests=num_requests, alpha=1.0, obj_size=obj_size, seed=42
         )
 
         hit_count = sum(1 for req in test_generator if cache.get(req))
         hit_ratio = hit_count / num_requests
         objects_fit = cache_size // obj_size
 
-        print(f"{cache_size//1024//1024}MB{'':<8} {hit_ratio:<12.3f} ~{objects_fit}")
+        print(f"{cache_size // 1024 // 1024}MB{'':<8} {hit_ratio:<12.3f} ~{objects_fit}")
 
     print("\nNotes:")
     print("- Higher α values create more skewed access patterns")