Merge branch 'main' into fork/zzstoatzz/main

Author: MarshalX

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
 
           - os: windows
             target: x86_64
-            interpreter: pypy3.9 pypy3.10
+            interpreter: pypy3.9 pypy3.10 pypy3.11
           - os: windows
             target: i686
             python-architecture: x86
@@ -36,15 +36,15 @@ jobs:
 
           - os: macos
             target: x86_64
-            interpreter: pypy3.9 pypy3.10
+            interpreter: pypy3.9 pypy3.10 pypy3.11
           - os: macos
             target: aarch64
             # actions/setup-python@v5 does not support 3.8 and 3.9 on arm64
-            interpreter: 3.8 3.9 pypy3.9 pypy3.10
+            interpreter: 3.8 3.9 pypy3.9 pypy3.10 pypy3.11
 
           - os: ubuntu
             target: x86_64
-            interpreter: pypy3.9 pypy3.10
+            interpreter: pypy3.9 pypy3.10 pypy3.11
           - os: ubuntu
             target: i686
           - os: ubuntu
@@ -90,7 +90,7 @@ jobs:
           target: ${{ matrix.target }}
           manylinux: ${{ matrix.manylinux || 'auto' }}
           container: ${{ matrix.container }}
-          args: --release --out dist --interpreter ${{ matrix.maturin-interpreter || matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10' }} ${{ matrix.extra-build-args }}
+          args: --release --out dist --interpreter ${{ matrix.maturin-interpreter || matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10 pypy3.11' }} ${{ matrix.extra-build-args }}
           rust-toolchain: 1.76.0
           docker-options: -e CI
 

Cargo.toml

@@ -5,16 +5,16 @@ edition = "2021"
 license = "MIT"
 description = "Python binding to the Rust IPLD library"
 authors = ["Ilya (Marshal) <ilya@marshal.dev>"]
-keywords = ["python", "binding", "library", "lib", "ipld", "cid", "multibase", "multihash", "dag", "cbor", "json", "pb", "dag-cbor", "dag-json"]
+keywords = ["ipld", "cid", "multibase", "multihash", "dag-cbor"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [lib]
 name = "libipld"
 crate-type = ["rlib", "cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.23.4", features = ["generate-import-lib", "anyhow"] }
-python3-dll-a = "0.2.13"
+pyo3 = { version = "0.25.1", features = ["generate-import-lib", "anyhow"] }
+python3-dll-a = "0.2.14"
 anyhow = "1.0.95"
 libipld = { version = "0.16.0", features = ["dag-cbor"] }
 multibase = "0.9.1"

profiling/Cargo.toml

@@ -11,4 +11,4 @@ structopt = "0.3.26"
 clap = "4.5.29"
 
 [dependencies.pyo3]
-version = "0.23.4"
+version = "0.25.1"

profiling/src/profiles/encode_dag_cbor.rs

@@ -14,7 +14,7 @@ pub fn exec(iterations: u64) {
 
     for _ in 0..iterations {
         Python::with_gil(|gil| {
-            println!("{}", libipld::encode_dag_cbor(gil, &PyString::new_bound(gil, json_str)).is_ok());
+            println!("{}", libipld::encode_dag_cbor(gil, &PyString::new(gil, json_str)).is_ok());
         });
     }
 }

pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 license = "MIT"
 repository = "https://github.com/MarshalX/python-libipld"
 readme = "README.md"
-keywords = ["library", "lib", "ipld", "cid", "multibase", "multihash", "dag", "cbor", "json", "pb", "dag-cbor", "dag-json"]
+keywords = ["library", "lib", "ipld", "cid", "multibase", "multihash", "dag", "cbor", "dag-cbor"]
 requires-python = ">=3.8"
 classifiers = [
     "Development Status :: 5 - Production/Stable",

pytests/test_dag_cbor.py

@@ -135,3 +135,95 @@ def test_recursion_limit_exceed_on_nested_maps() -> None:
         libipld.decode_dag_cbor(dag_cbor)
 
     assert 'in DAG-CBOR decoding' in str(exc_info.value)
+
+
+def test_dag_cbor_decode_largest_unsigned_int_roundtrip() -> None:
+    data = bytes.fromhex('1bffffffffffffffff')
+
+    decoded_result = libipld.decode_dag_cbor(data)
+    assert decoded_result == 2**64 - 1
+
+    encoded_result = libipld.encode_dag_cbor(decoded_result)
+    assert encoded_result == data
+
+
+def test_dag_cbor_decode_smallest_negative_int_roundtrip() -> None:
+    data = bytes.fromhex('3bffffffffffffffff')
+
+    decoded_result = libipld.decode_dag_cbor(data)
+    assert decoded_result == -(2**64)
+
+    encoded_result = libipld.encode_dag_cbor(decoded_result)
+    assert encoded_result == data
+
+
+def test_dag_cbor_decode_invalid_utf8() -> None:
+    with pytest.raises(ValueError) as exc_info:
+        libipld.decode_dag_cbor(bytes.fromhex('62c328'))
+
+
+    assert 'Invalid UTF-8 string' in str(exc_info.value)
+
+
+def test_dab_cbor_decode_map_int_key() -> None:
+    dag_cbor = bytes.fromhex('a10000')
+    with pytest.raises(ValueError) as exc_info:
+        libipld.decode_dag_cbor(dag_cbor)
+
+    assert 'Map keys must be strings' in str(exc_info.value)
+
+
+def test_dab_cbor_encode_map_int_key() -> None:
+    obj = {0: 'value'}
+    with pytest.raises(ValueError) as exc_info:
+        libipld.encode_dag_cbor(obj)
+
+    assert 'Map keys must be strings' in str(exc_info.value)
+
+
+def test_dag_cbor_decode_nan_f64_error() -> None:
+    # fb7ff8000000000000 - IEEE 754 double precision NaN
+    with pytest.raises(ValueError) as exc_info:
+        libipld.decode_dag_cbor(bytes.fromhex('fb7ff8000000000000'))
+
+    assert 'number out of range for f64' in str(exc_info.value).lower()
+
+
+def test_dag_cbor_decode_positive_infinity_f64_error() -> None:
+    # fb7ff0000000000000 - IEEE 754 double precision positive infinity
+    with pytest.raises(ValueError) as exc_info:
+        libipld.decode_dag_cbor(bytes.fromhex('fb7ff0000000000000'))
+
+    assert 'number out of range for f64' in str(exc_info.value).lower()
+
+
+def test_dag_cbor_decode_negative_infinity_f64_error() -> None:
+    # fbfff0000000000000 - IEEE 754 double precision negative infinity
+    with pytest.raises(ValueError) as exc_info:
+        libipld.decode_dag_cbor(bytes.fromhex('fbfff0000000000000'))
+
+    assert 'number out of range for f64' in str(exc_info.value).lower()
+
+
+def test_dag_cbor_decode_nan_f32_error() -> None:
+    # fa7fc00000 - IEEE 754 single precision NaN
+    with pytest.raises(ValueError) as exc_info:
+        libipld.decode_dag_cbor(bytes.fromhex('fa7fc00000'))
+
+    assert 'number out of range for f32' in str(exc_info.value).lower()
+
+
+def test_dag_cbor_decode_positive_infinity_f32_error() -> None:
+    # fa7f800000 - IEEE 754 single precision positive infinity
+    with pytest.raises(ValueError) as exc_info:
+        libipld.decode_dag_cbor(bytes.fromhex('fa7f800000'))
+
+    assert 'number out of range for f32' in str(exc_info.value).lower()
+
+
+def test_dag_cbor_decode_negative_infinity_f32_error() -> None:
+    # faff800000 - IEEE 754 single precision negative infinity
+    with pytest.raises(ValueError) as exc_info:
+        libipld.decode_dag_cbor(bytes.fromhex('faff800000'))
+
+    assert 'number out of range for f32' in str(exc_info.value).lower()

src/lib.rs

@@ -51,18 +51,24 @@ fn map_key_cmp(a: &Vec<u8>, b: &Vec<u8>) -> std::cmp::Ordering {
     }
 }
 
-fn sort_map_keys(keys: &Bound<PyList>, len: usize) -> Vec<(PyBackedStr, usize)> {
+fn sort_map_keys(keys: &Bound<PyList>, len: usize) -> Result<Vec<(PyBackedStr, usize)>> {
     // Returns key and index.
     let mut keys_str = Vec::with_capacity(len);
     for i in 0..len {
-        let item = keys.get_item(i).unwrap();
-        let key = item.downcast::<PyString>().unwrap().to_owned();
-        let backed_str = PyBackedStr::try_from(key).unwrap();
+        let item = keys.get_item(i)?;
+        let key = match item.downcast::<PyString>() {
+            Ok(k) => k.to_owned(),
+            Err(_) => return Err(anyhow!("Map keys must be strings")),
+        };
+        let backed_str = match PyBackedStr::try_from(key) {
+            Ok(bs) => bs,
+            Err(_) => return Err(anyhow!("Failed to convert PyString to PyBackedStr")),
+        };
         keys_str.push((backed_str, i));
     }
 
     if keys_str.len() < 2 {
-        return keys_str;
+        return Ok(keys_str);
     }
 
     keys_str.sort_by(|a, b| {
@@ -78,7 +84,7 @@ fn sort_map_keys(keys: &Bound<PyList>, len: usize) -> Vec<(PyBackedStr, usize)>
         }
     });
 
-    keys_str
+    Ok(keys_str)
 }
 
 fn get_bytes_from_py_any<'py>(obj: &'py Bound<'py, PyAny>) -> PyResult<&'py [u8]> {
@@ -96,11 +102,13 @@ fn get_bytes_from_py_any<'py>(obj: &'py Bound<'py, PyAny>) -> PyResult<&'py [u8]
     }
 }
 
-fn string_new_bound<'py>(py: Python<'py>, s: &[u8]) -> Bound<'py, PyString> {
+fn string_new_bound<'py>(py: Python<'py>, s: &[u8]) -> Result<Bound<'py, PyString>> {
+    std::str::from_utf8(s).map_err(|e| anyhow!("Invalid UTF-8 string: {}", e))?;
+
     let ptr = s.as_ptr() as *const c_char;
     let len = s.len() as ffi::Py_ssize_t;
     unsafe {
-        Bound::from_owned_ptr(py, ffi::PyUnicode_FromStringAndSize(ptr, len)).downcast_into_unchecked()
+        Ok(Bound::from_owned_ptr(py, ffi::PyUnicode_FromStringAndSize(ptr, len)).downcast_into_unchecked())
     }
 }
 
@@ -122,14 +130,14 @@ fn decode_dag_cbor_to_pyobject<R: Read + Seek>(
     let major = decode::read_major(r)?;
     Ok(match major.kind() {
         MajorKind::UnsignedInt => decode::read_uint(r, major)?.into_pyobject(py)?.into(),
-        MajorKind::NegativeInt => (-1 - decode::read_uint(r, major)? as i64).into_pyobject(py)?.into(),
+        MajorKind::NegativeInt => (-1 - decode::read_uint(r, major)? as i128).into_pyobject(py)?.into(),
         MajorKind::ByteString => {
             let len = decode::read_uint(r, major)?;
             PyBytes::new(py, &decode::read_bytes(r, len)?).into_pyobject(py)?.into()
         }
         MajorKind::TextString => {
             let len = decode::read_uint(r, major)?;
-            string_new_bound(py, &decode::read_bytes(r, len)?).into_pyobject(py)?.into()
+            string_new_bound(py, &decode::read_bytes(r, len)?)?.into_pyobject(py)?.into()
         }
         MajorKind::Array => {
             let len: ffi::Py_ssize_t = decode_len(decode::read_uint(r, major)?)?.try_into()?;
@@ -167,7 +175,7 @@ fn decode_dag_cbor_to_pyobject<R: Read + Seek>(
                     }
                 }
 
-                let key_py = string_new_bound(py, key.as_slice()).into_pyobject(py)?;
+                let key_py = string_new_bound(py, key.as_slice())?.into_pyobject(py)?;
                 prev_key = Some(key);
 
                 let value_py = decode_dag_cbor_to_pyobject(py, r, depth + 1)?;
@@ -191,8 +199,20 @@ fn decode_dag_cbor_to_pyobject<R: Read + Seek>(
             cbor::FALSE => false.into_pyobject(py)?.into_any().unbind(),
             cbor::TRUE => true.into_pyobject(py)?.into_any().unbind(),
             cbor::NULL => py.None(),
-            cbor::F32 => decode::read_f32(r)?.into_pyobject(py)?.into(),
-            cbor::F64 => decode::read_f64(r)?.into_pyobject(py)?.into(),
+            cbor::F32 => {
+                let value = decode::read_f32(r)?;
+                if !value.is_finite() {
+                    return Err(anyhow!("Number out of range for f32 (NaNs are forbidden)".to_string()));
+                }
+                value.into_pyobject(py)?.into()
+            },
+            cbor::F64 => {
+                let value = decode::read_f64(r)?;
+                if !value.is_finite() {
+                    return Err(anyhow!("Number out of range for f64 (NaNs are forbidden)".to_string()));
+                }
+                value.into_pyobject(py)?.into()
+            },
             _ => return Err(anyhow!("Unsupported major type".to_string())),
         },
     })
@@ -231,7 +251,7 @@ fn encode_dag_cbor_from_pyobject<'py, W: Write>(
 
         Ok(())
     } else if obj.is_instance_of::<PyInt>() {
-        let i: i64 = obj.extract()?;
+        let i: i128 = obj.extract()?;
 
         if i.is_negative() {
             encode::write_u64(w, MajorKind::NegativeInt, -(i + 1) as u64)?
@@ -252,7 +272,7 @@ fn encode_dag_cbor_from_pyobject<'py, W: Write>(
         Ok(())
     } else if let Ok(map) = obj.downcast::<PyDict>() {
         let len = map.len();
-        let keys = sort_map_keys(&map.keys(), len);
+        let keys = sort_map_keys(&map.keys(), len)?;
         let values = map.values();
 
         encode::write_u64(w, MajorKind::Map, len as u64)?;