Merge branch 'main' into test-panic-abort

Author: MarshalX

.github/workflows/benchmark.yml

@@ -0,0 +1,55 @@
+name: Benchmark
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  UV_FROZEN: true
+  UV_PYTHON: 3.14
+  RUST_VERSION: "1.90.0"
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository.
+        uses: actions/checkout@v4
+
+      - name: Install UV.
+        uses: astral-sh/setup-uv@v6
+
+      - name: Install dependencies.
+        run: uv sync --group testing
+
+      - name: Compile.
+        run: uv pip install -v -e .
+        env:
+          RUST_BACKTRACE: 1
+
+      - name: Run Benchmarks.
+        run: uv run pytest . -m benchmark_main --benchmark-enable --benchmark-json=output.json
+
+      - name: Download previous benchmark data.
+        uses: actions/cache@v4
+        with:
+          path: ./benchmark-data
+          key: benchmark-${{ github.base_ref || github.ref_name }}
+
+      - name: Benchmark results.
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'pytest'
+          output-file-path: output.json
+          external-data-json-path: ./benchmark-data/output.json
+          summary-always: true
+          fail-on-alert: true
+          skip-fetch-gh-pages: true
+          save-data-file: ${{ github.event_name != 'pull_request' }}

.github/workflows/codspeed.yml

@@ -12,7 +12,7 @@ permissions:
 
 env:
   UV_FROZEN: true
-  UV_PYTHON: 3.13 # use the latest version of Python because it is faster FIXME(@MarshalX): update to 3.14 when CodSpeed supports it
+  UV_PYTHON: 3.14
   RUST_VERSION: "1.90.0"
 
 jobs:
@@ -79,6 +79,6 @@ jobs:
       - name: Run CodSpeed benchmarks.
         uses: CodSpeedHQ/action@v4
         with:
-          mode: instrumentation
+          mode: simulation
           token: ${{ secrets.CODSPEED_TOKEN }}
-          run: uv run --group=codspeed pytest . --codspeed -n auto
+          run: uv run --group=codspeed pytest . --codspeed

.github/workflows/lint.yml

@@ -0,0 +1,23 @@
+name: Lint and format check
+
+on: [ pull_request ]
+
+permissions:
+  contents: read
+
+env:
+  RUSTFLAGS: "-Dwarnings"  # treat warnings as errors
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository.
+        uses: actions/checkout@v6
+
+      - name: Run Clippy.
+        run: cargo clippy --all-targets --all-features
+
+      - name: Run fmt check.
+        run: cargo fmt --all -- --check

CONTRIBUTING.md

@@ -0,0 +1,80 @@
+## Contibuting to python-libipld
+
+This project is a small, single-file wrapper around Rust crates like `cid`, `cbor4ii`, and `multibase`, exposing a Python API through `PyO3`. Despite its size, performance matters a lot.
+
+The project uses `uv` package manager. Installing UV: https://docs.astral.sh/uv/getting-started/installation/
+
+Commands for quick start:
+```shell
+# install deps
+uv sync --group all
+
+# compile and install using maturin directly (faster and better for developing)
+uv run maturin develop
+
+# compile and install using pip and maturin backend
+uv pip install -v -e .
+
+# run all tests
+uv run pytest
+
+# run the most important benchmarks
+uv run pytest . -m benchmark_main --benchmark-enable
+
+# run lint and fmt
+cargo clippy && cargo fmt
+```
+
+### Performance
+
+Two key points:
+
+1. Python-side benchmarks
+
+    We use `pytest-benchmark` and run all benchmarks from the Python side. `CodSpeed` is used in CI/CD, but it relies on CPU simulation. The best comparison is always on your local machine.
+
+    First, capture the baseline from the `main` branch. This records performance relative to your hardware:
+    ```shell
+    # clone and checkout main branch
+    uv pip install -v -e .  
+    # run the most important benchmarks
+    uv run pytest . -m benchmark_main --benchmark-enable --benchmark-save=main
+    ```
+
+    Then, on your feature branch, run the same benchmarks but save under a different name (`--benchmark-save` argument)
+    ```shell
+    # checkout your branch
+    uv pip install -v -e .
+    uv run pytest . -m benchmark_main --benchmark-enable --benchmark-save=your_feature
+    ```
+
+    Finally, compare results:
+    ```shell
+    uv run pytest-benchmark compare --group-by="name" 
+    ```
+
+    Notes:
+    - Benchmark data is stored under `.benchmarks`.
+    - You can delete old snapshots during local development.
+
+2. Rust-side benchmarks
+
+    We also maintain Rust benchmarks, but they mainly exist for profiling and diagnosing performance issues. They work better with tools like flamegraph than when forced into a Python boundary. See the project's [Makefile](Makefile) for details.
+
+### Testing
+
+All tests target the Python-facing API, which is why the `pytest` directory exists.
+
+Any segfaults or Rust panics **must** be handled safely and must never crash the Python interpreter. Every error must be catchable at the Python layer.
+
+### Style
+
+Use `cargo fmt` and `cargo clippy`. CI will block your PR if formatting or linting fails.
+
+### Things to care about
+
+This library is used in:
+- DAG-CBOR benchmarks for Python: https://github.com/DavidBuchanan314/dag-cbor-benchmark
+- DASL Testing: https://hyphacoop.github.io/dasl-testing/
+
+Keep these in mind and consider running their test suites against your feature branch locally.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "libipld"
-version = "3.1.1"
+version = "3.2.0"
 edition = "2021"
 license = "MIT"
 description = "Python binding to the Rust IPLD library"
@@ -16,10 +16,8 @@ crate-type = ["rlib", "cdylib"]
 pyo3 = { version = "0.27.1", features = ["generate-import-lib", "anyhow"] }
 python3-dll-a = "0.2.14"
 anyhow = "1.0.100"
-libipld = { version = "0.16.0", features = ["dag-cbor"] }
-multibase = "0.9.2"
-byteorder = "1.5.0"
-multihash = "0.18.1"
+cid = "0.11.1"
+cbor4ii = { version = "1.2.1" }
 
 [workspace]
 members = [ "profiling" ]

profiling/src/profiles/encode_dag_cbor.rs

@@ -14,7 +14,10 @@ pub fn exec(iterations: u64) {
 
     for _ in 0..iterations {
         Python::with_gil(|gil| {
-            println!("{}", libipld::encode_dag_cbor(gil, &PyString::new(gil, json_str)).is_ok());
+            println!(
+                "{}",
+                libipld::encode_dag_cbor(gil, &PyString::new(gil, json_str)).is_ok()
+            );
         });
     }
 }

profiling/src/profiles/mod.rs

@@ -1,3 +1,3 @@
 pub mod decode_car;
-pub mod encode_dag_cbor;
 pub mod decode_dag_cbor;
+pub mod encode_dag_cbor;

pyproject.toml

@@ -50,8 +50,8 @@ testing = [
     'pytest-xdist==3.8.0; python_version >= "3.9"',
 ]
 codspeed = [
-    # only run on CI with the latest Python version FIXME(@MarshalX): update to 3.14 when CodSpeed supports it
-    'pytest-codspeed==4.1.1; python_version == "3.13" and implementation_name == "cpython"',
+    # only run on CI with the latest Python version
+    'pytest-codspeed==4.2.0; python_version == "3.14" and implementation_name == "cpython"',
 ]
 
 all = [
@@ -60,6 +60,9 @@ all = [
 ]
 
 [tool.pytest.ini_options]
+markers = [
+    'benchmark_main: marks tests as main benchmarks to run selectively'
+]
 addopts = [
     '--benchmark-columns', 'min,mean,stddev,outliers,rounds,iterations',
     '--benchmark-disable',  # use --benchmark-enable

pytests/test_dag_cbor.py

@@ -101,11 +101,13 @@ def test_dag_cbor_decode(benchmark, data) -> None:
     _dag_cbor_roundtrip(benchmark, data)
 
 
+@pytest.mark.benchmark_main
 @pytest.mark.parametrize('data', load_json_data_fixtures(_REAL_DATA_DIR), ids=lambda data: data[0])
 def test_dag_cbor_encode_real_data(benchmark, data) -> None:
     _dag_cbor_encode(benchmark, data)
 
 
+@pytest.mark.benchmark_main
 @pytest.mark.parametrize('data', load_json_data_fixtures(_REAL_DATA_DIR), ids=lambda data: data[0])
 def test_dag_cbor_decode_real_data(benchmark, data) -> None:
     _dag_cbor_roundtrip(benchmark, data)
@@ -116,6 +118,7 @@ def test_dag_cbor_decode_fixtures(benchmark, data) -> None:
     _dag_cbor_decode(benchmark, data)
 
 
+@pytest.mark.benchmark_main
 def test_dag_cbor_decode_torture_cids(benchmark) -> None:
     dag_cbor = open(_TORTURE_CIDS_DAG_CBOR_PATH, 'rb').read()
     benchmark(libipld.decode_dag_cbor, dag_cbor)
@@ -162,7 +165,7 @@ def test_dag_cbor_decode_invalid_utf8() -> None:
         libipld.decode_dag_cbor(bytes.fromhex('62c328'))
 
 
-    assert 'Invalid UTF-8 string' in str(exc_info.value)
+    assert 'utf-8' in str(exc_info.value)
 
 
 def test_dab_cbor_decode_map_int_key() -> None:

pytests/test_decode_car.py

@@ -14,6 +14,7 @@ def car() -> bytes:
     return load_car_fixture(_DID, _REPO_CAR_PATH)
 
 
+@pytest.mark.benchmark_main
 def test_decode_car(benchmark, car) -> None:
     header, blocks = benchmark(libipld.decode_car, car)