Merge branch 'main' into dev

Author: sonam-pankaj95

Cargo.lock

@@ -20,10 +20,10 @@ version = "0.7.0"
 
 [workspace.dependencies]
 pdf-extract = "0.9.0"
-candle-nn = { version = "0.9.1" }
-candle-transformers = { version = "0.9.1" }
-candle-core = { version = "0.9.1" }
-candle-flash-attn = { version = "0.9.1" }
+candle-nn = { version = "0.9.2" }
+candle-transformers = { version = "0.9.2" }
+candle-core = { version = "0.9.2" }
+candle-flash-attn = { version = "0.9.2" }
 processors-rs = { path = "processors", version = "=0.7.0" }
 
 strum = "0.27.1"

Cargo.toml

@@ -17,6 +17,7 @@ strum =  {workspace = true}
 strum_macros =  {workspace = true}
 
 [features]
+default = ["extension-module", "ort", "audio", "aws"]
 extension-module = ["pyo3/extension-module"]
 mkl = ["embed_anything/mkl"]
 accelerate = ["embed_anything/accelerate"]
@@ -25,4 +26,4 @@ cudnn = ["embed_anything/cudnn"]
 metal = ["embed_anything/metal"]
 ort = ["embed_anything/ort"]
 audio = ["embed_anything/audio"]
-video = ["embed_anything/video"]
+aws = ["embed_anything/aws"]

python/Cargo.toml

@@ -0,0 +1,22 @@
+# embed_anything Python extension
+
+This crate is the Rust/PyO3 extension for the `embed_anything` Python package. It is **not** built with plain `cargo`.
+
+## Building
+
+From the **repository root** (not from this directory), use maturin so the extension links against your Python installation:
+
+```bash
+# From repository root
+maturin develop    # build and install in the current environment (editable)
+# or
+maturin build      # build a wheel
+```
+
+Or install the package (maturin is the build backend in `pyproject.toml`):
+
+```bash
+pip install -e .
+```
+
+Do **not** run `cargo build` in this directory or `cargo build -p embed_anything_python` from the root—the linker will fail with undefined Python symbols because cargo does not link libpython.

python/README.md

@@ -1,5 +1,6 @@
 pub mod config;
 pub mod models;
+#[cfg(feature = "aws")]
 pub mod s3_client;
 use embed_anything::embeddings::embed::{TextEmbedder, VisionEmbedder};
 use embed_anything::{

python/src/lib.rs

@@ -30,7 +30,7 @@ rayon = "1.8.1"
 
 # Natural Language Processing
 tokenizers = {version="0.22.1", default-features = false, features=["http"]}
-text-splitter = {version= "0.28.0", features=["tokenizers", "markdown"]}
+text-splitter = {version= "0.29.0", features=["tokenizers", "markdown"]}
 
 tracing = "0.1.41"
 
@@ -47,7 +47,7 @@ anyhow = "1.0.98"
 tokio = { version = "1.40.0", features = ["macros", "rt-multi-thread"] }
 
 chrono = "0.4.38"
-rand = "0.9.0"
+rand = "0.10.0"
 itertools = "0.14.0"
 
 # Image processing
@@ -72,9 +72,9 @@ model2vec-rs = "0.1.4"
 
 # Logging
 log = "0.4"
-aws-sdk-s3 = {version = "1.110.0", features = ["rt-tokio"]}
-aws-config = {version = "1.8.10", features = ["behavior-version-latest"]} 
-aws-credential-types = "1.2.8"
+aws-sdk-s3 = {version = "1.110.0", features = ["rt-tokio"], optional= true}
+aws-config = {version = "1.8.10", features = ["behavior-version-latest"], optional = true} 
+aws-credential-types = {version  = "1.2.8", optional = true}
 
 [dev-dependencies]
 tempdir = "0.3.7"
@@ -98,6 +98,7 @@ metal = ["candle-core/metal", "candle-nn/metal"]
 audio = ["dep:symphonia"]
 video = ["processors-rs/video"]
 ort = ["dep:ort",]
+aws = ["dep:aws-sdk-s3", "dep:aws-config", "dep:aws-credential-types"]
 rustls-tls = [
     "reqwest/rustls-tls",
     "hf-hub/rustls-tls",

rust/Cargo.toml

@@ -67,6 +67,33 @@
 //! # Ok(())
 //! # }
 //! ```
+//! //! # Feature flags
+//!
+//! The crate can be configured with the following [Cargo features](https://doc.rust-lang.org/cargo/reference/features.html):
+//!
+//! | Feature | Description |
+//! |---------|-------------|
+//! | **default** | Enables `rustls-tls` for TLS. Keep this unless you need a different TLS backend. |
+//! | **rustls-tls** | Use [Rustls](https://github.com/rustls/rustls) for TLS in HTTP clients (reqwest, hf-hub, tokenizers). |
+//! | **mkl** | Intel [MKL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html) (Math Kernel Library) for CPU-accelerated linear algebra. |
+//! | **accelerate** | Apple [Accelerate](https://developer.apple.com/documentation/accelerate) framework for optimized math on macOS. |
+//! | **cuda** | [NVIDIA CUDA](https://developer.nvidia.com/cuda-toolkit) support for GPU-accelerated inference. |
+//! | **cudnn** | [cuDNN](https://developer.nvidia.com/cudnn) for CUDA-accelerated deep learning (enables `cuda`-related optimizations). |
+//! | **flash-attn** | [Flash Attention](https://github.com/Dao-AILab/flash-attention) for faster attention on CUDA GPUs. Implies **cuda**. |
+//! | **metal** | Apple [Metal](https://developer.apple.com/metal/) for GPU-accelerated inference on macOS. |
+//! | **audio** | Audio embedding via [Symphonia](https://github.com/pdeljanov/Symphonia): transcribe and embed audio files (e.g. `emb_audio`). |
+//! | **ort** | [ONNX Runtime](https://onnxruntime.ai/) for running ONNX models (e.g. reranker and ONNX-based embedders). |
+//! | **aws** | [AWS SDK](https://aws.amazon.com/sdk-for-rust/) for loading objects from S3 (e.g. `s3_loader` module). |
+//!
+//! ## Example
+//!
+//! Enable only the features you need to reduce build time and binary size:
+//!
+//! ```toml
+//! [dependencies]
+//! embed_anything = { path = "rust", default-features = false, features = ["rustls-tls", "audio", "aws"] }
+//! ```
+//!
 
 pub mod chunkers;
 pub mod config;
@@ -77,6 +104,7 @@ pub mod models;
 #[cfg(feature = "ort")]
 pub mod reranker;
 pub mod text_loader;
+#[cfg(feature = "aws")]
 pub mod s3_loader;
 
 use anyhow::{Error, Result};

rust/src/lib.rs

@@ -55,6 +55,13 @@ impl CustomOp1 for NonZero {
             CpuStorage::F16(vs) => self.nonzero(vs, layout),
             CpuStorage::F32(vs) => self.nonzero(vs, layout),
             CpuStorage::F64(vs) => self.nonzero(vs, layout),
+            CpuStorage::I16(vs) => self.nonzero(vs, layout),
+            CpuStorage::I32(vs) => self.nonzero(vs, layout),
+            CpuStorage::F8E4M3(vs) => self.nonzero(vs, layout),
+            CpuStorage::F6E2M3(vs) => self.nonzero(vs, layout),
+            CpuStorage::F6E3M2(vs) => self.nonzero(vs, layout),
+            CpuStorage::F4(vs) => self.nonzero(vs, layout),
+            CpuStorage::F8E8M0(vs) => self.nonzero(vs, layout)
         };
         let index_len = layout.dims().len();
         let result_len = result.len() / index_len;