Merge origin/main into pr-34

Author: Pringled

.github/workflows/ci.yaml

@@ -12,9 +12,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        backend: # Different backends for Tokenizers
-          - default  
-          - fancy-regex     
+        include:
+          - name: default
+            build: cargo build --verbose
+            test: cargo test --verbose
+          - name: fancy-regex
+            build: cargo build --no-default-features --features fancy-regex --verbose
+            test: cargo test --no-default-features --features fancy-regex --verbose
+          - name: local-only
+            build: cargo build --no-default-features --features onig,local-only --verbose
+            test: cargo test --no-default-features --features onig,local-only --verbose
 
     steps:
       - name: Checkout
@@ -26,23 +33,28 @@ jobs:
           components: clippy, rustfmt
 
       - name: Build
-        run: |
-          if [ "${{ matrix.backend }}" = "default" ]; then
-            cargo build --verbose
-          else
-            cargo build --no-default-features --features fancy-regex --verbose
-          fi
+        run: ${{ matrix.build }}
 
       - name: Test
-        run: |
-          if [ "${{ matrix.backend }}" = "default" ]; then
-            cargo test --verbose
-          else
-            cargo test --no-default-features --features fancy-regex --verbose
-          fi
+        run: ${{ matrix.test }}
 
       - name: Lint (clippy)
         run: cargo clippy --all-targets --all-features -- -D warnings
 
       - name: Format check
         run: cargo fmt -- --check
+
+  wasm-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          target: wasm32-unknown-unknown
+
+      - name: Check wasm feature set
+        run: RUSTFLAGS='--cfg getrandom_backend="wasm_js"' cargo check --no-default-features --features wasm --target wasm32-unknown-unknown

Cargo.lock

@@ -14,23 +14,25 @@ categories = ["science", "text-processing"]
 exclude = ["tests/*"]
 
 [features]
-default      = ["onig"]
+default      = ["onig", "hf-hub"]
+hf-hub       = ["dep:hf-hub", "dep:ureq"]
+local-only   = []
 onig         = ["tokenizers/onig",
                 "tokenizers/progressbar",
-                "tokenizers/esaxx_fast",
-                "tokenizers/http"]
+                "tokenizers/esaxx_fast"]
 
 fancy-regex  = ["tokenizers/fancy-regex",
                 "tokenizers/progressbar",
-                "tokenizers/esaxx_fast",
-                "tokenizers/http"]
+                "tokenizers/esaxx_fast"]
+wasm         = ["local-only",
+                "tokenizers/unstable_wasm"]
 
 [dependencies]
 tokenizers = { version = "0.21", default-features = false }
 safetensors = "0.5"
 ndarray = "0.15"
-hf-hub = { version = "0.4", default-features = false, features = ["ureq"] }
-ureq = "2"
+hf-hub = { version = "0.4", default-features = false, features = ["ureq"], optional = true }
+ureq = { version = "2", optional = true }
 clap = { version = "4.0", features = ["derive"] }
 anyhow = "1.0"
 serde_json = "1.0"

Cargo.toml

@@ -145,6 +145,36 @@ cargo build --release
 *   **Batch Processing:** Encodes multiple sentences in batches.
 *   **Configurable Encoding:** Allows customization of maximum sequence length and batch size during encoding.
 
+### Feature flags
+
+The crate exposes a few feature combinations for different runtimes:
+
+* `default`: native build with `onig` tokenization and optional Hugging Face Hub downloads
+* `fancy-regex`: alternative tokenizer backend for native builds
+* `local-only`: disable remote model downloads and restrict loading to local paths or `from_bytes(...)`
+* `wasm`: minimal WebAssembly-oriented feature set for in-memory loading via `from_bytes(...)`
+
+Typical invocations are:
+
+* native local-only build:
+  `cargo build --no-default-features --features onig,local-only`
+* wasm check:
+  `RUSTFLAGS='--cfg getrandom_backend="wasm_js"' cargo check --no-default-features --features wasm --target wasm32-unknown-unknown`
+
+The `wasm` feature is intended for `wasm32-unknown-unknown` builds that load models
+from in-memory bytes, for example after fetching assets over HTTP or embedding them
+into the binary. Direct filesystem access is usually not available in browser-style
+WebAssembly environments, so callers should pass file contents through `from_bytes(...)`.
+Remote Hugging Face downloads are not available in this mode.
+
+For `wasm32-unknown-unknown`, `getrandom` also requires a target-specific backend
+configuration. The minimal check command is:
+
+```bash
+RUSTFLAGS='--cfg getrandom_backend="wasm_js"' \
+cargo check --no-default-features --features wasm --target wasm32-unknown-unknown
+```
+
 ## What is Model2Vec?
 
 Model2Vec is a technique to distill large sentence transformer models into highly efficient static embedding models. This process significantly reduces model size and computational requirements for inference. For a detailed understanding of how Model2Vec works, including the distillation process and model training, please refer to the [main Model2Vec Python repository](https://github.com/MinishLab/model2vec) and its [documentation](https://github.com/MinishLab/model2vec/blob/main/docs/what_is_model2vec.md).