Update dependencies and versions in Cargo files and add Python README

- Updated versions for several dependencies in Cargo.lock and Cargo.toml, including candle packages and aws SDKs, to enhance functionality and compatibility.
- Introduced a new README.md for the Python extension, detailing build instructions and usage guidelines.
- Added AWS feature support in Rust and Python modules to improve cloud integration capabilities.

Author: akshayballal95

Cargo.lock

@@ -20,10 +20,10 @@ version = "0.7.0"
 
 [workspace.dependencies]
 pdf-extract = "0.9.0"
-candle-nn = { version = "0.9.1" }
-candle-transformers = { version = "0.9.1" }
-candle-core = { version = "0.9.1" }
-candle-flash-attn = { version = "0.9.1" }
+candle-nn = { version = "0.9.2" }
+candle-transformers = { version = "0.9.2" }
+candle-core = { version = "0.9.2" }
+candle-flash-attn = { version = "0.9.2" }
 processors-rs = { path = "processors", version = "=0.7.0" }
 
 strum = "0.27.1"

Cargo.toml

@@ -17,6 +17,7 @@ strum =  {workspace = true}
 strum_macros =  {workspace = true}
 
 [features]
+default = ["extension-module", "ort", "audio", "aws"]
 extension-module = ["pyo3/extension-module"]
 mkl = ["embed_anything/mkl"]
 accelerate = ["embed_anything/accelerate"]
@@ -25,3 +26,4 @@ cudnn = ["embed_anything/cudnn"]
 metal = ["embed_anything/metal"]
 ort = ["embed_anything/ort"]
 audio = ["embed_anything/audio"]
+aws = ["embed_anything/aws"]

python/Cargo.toml

@@ -0,0 +1,22 @@
+# embed_anything Python extension
+
+This crate is the Rust/PyO3 extension for the `embed_anything` Python package. It is **not** built with plain `cargo`.
+
+## Building
+
+From the **repository root** (not from this directory), use maturin so the extension links against your Python installation:
+
+```bash
+# From repository root
+maturin develop    # build and install in the current environment (editable)
+# or
+maturin build      # build a wheel
+```
+
+Or install the package (maturin is the build backend in `pyproject.toml`):
+
+```bash
+pip install -e .
+```
+
+Do **not** run `cargo build` in this directory or `cargo build -p embed_anything_python` from the root—the linker will fail with undefined Python symbols because cargo does not link libpython.

python/README.md

@@ -1,5 +1,6 @@
 pub mod config;
 pub mod models;
+#[cfg(feature = "aws")]
 pub mod s3_client;
 use embed_anything::embeddings::embed::{TextEmbedder, VisionEmbedder};
 use embed_anything::{

python/src/lib.rs

@@ -72,9 +72,9 @@ model2vec-rs = "0.1.4"
 
 # Logging
 log = "0.4"
-aws-sdk-s3 = {version = "1.110.0", features = ["rt-tokio"]}
-aws-config = {version = "1.8.10", features = ["behavior-version-latest"]} 
-aws-credential-types = "1.2.8"
+aws-sdk-s3 = {version = "1.110.0", features = ["rt-tokio"], optional= true}
+aws-config = {version = "1.8.10", features = ["behavior-version-latest"], optional = true} 
+aws-credential-types = {version  = "1.2.8", optional = true}
 
 [dev-dependencies]
 tempdir = "0.3.7"
@@ -97,6 +97,7 @@ flash-attn = ["cuda", "candle-transformers/flash-attn", "dep:candle-flash-attn"]
 metal = ["candle-core/metal", "candle-nn/metal"]
 audio = ["dep:symphonia"]
 ort = ["dep:ort",]
+aws = ["dep:aws-sdk-s3", "dep:aws-config", "dep:aws-credential-types"]
 rustls-tls = [
     "reqwest/rustls-tls",
     "hf-hub/rustls-tls",

rust/Cargo.toml

@@ -77,6 +77,7 @@ pub mod models;
 #[cfg(feature = "ort")]
 pub mod reranker;
 pub mod text_loader;
+#[cfg(feature = "aws")]
 pub mod s3_loader;
 
 use anyhow::{Error, Result};

rust/src/lib.rs

@@ -55,6 +55,13 @@ impl CustomOp1 for NonZero {
             CpuStorage::F16(vs) => self.nonzero(vs, layout),
             CpuStorage::F32(vs) => self.nonzero(vs, layout),
             CpuStorage::F64(vs) => self.nonzero(vs, layout),
+            CpuStorage::I16(vs) => self.nonzero(vs, layout),
+            CpuStorage::I32(vs) => self.nonzero(vs, layout),
+            CpuStorage::F8E4M3(vs) => self.nonzero(vs, layout),
+            CpuStorage::F6E2M3(vs) => self.nonzero(vs, layout),
+            CpuStorage::F6E3M2(vs) => self.nonzero(vs, layout),
+            CpuStorage::F4(vs) => self.nonzero(vs, layout),
+            CpuStorage::F8E8M0(vs) => self.nonzero(vs, layout)
         };
         let index_len = layout.dims().len();
         let result_len = result.len() / index_len;