feat: add support for INT8 encoding and sparse vector indexing in vector API

Author: tae898

bindings/python/docs/api/database.md

@@ -648,10 +648,12 @@ db.create_vector_index(
     vertex_type: str,
     vector_property: str,
     dimensions: int,
+    id_property: str | None = None,
     distance_function: str = "cosine",
     max_connections: int = 16,
     beam_width: int = 100,
     quantization: str = "INT8",
+    encoding: str | None = None,
     location_cache_size: int | None = None,
     graph_build_cache_size: int | None = None,
     mutations_before_rebuild: int | None = None,
@@ -679,6 +681,7 @@ specifically need that surface.
 - `vertex_type` (str): Vertex type containing vectors
 - `vector_property` (str): Property storing vector arrays
 - `dimensions` (int): Vector dimensionality
+- `id_property` (str | None): Optional property used for key-based vector lookup.
 - `distance_function` (str): `"cosine"`, `"euclidean"`, or `"inner_product"`
 - `max_connections` (int): Max connections per node (default: 16). Maps to
   `maxConnections` in HNSW (JVector).
@@ -690,6 +693,10 @@ specifically need that surface.
   production workloads. In current ArcadeDB engine builds, `"PRODUCT"` also requires
   enough indexed vectors per bucket for PQ training. For tiny corpora, set `pq_clusters`
   explicitly to a small value or prefer another quantization mode.
+- `encoding` (str | None): Optional storage encoding for the document property.
+    Use `"INT8"` when the underlying property is `BINARY` and stores pre-quantized
+    bytes. Do not combine `encoding="INT8"` with `quantization="INT8"`; use
+    `quantization="NONE"` for native INT8 storage.
 - `location_cache_size` (int | None): Override location cache size (default: `None`, uses engine default).
 - `graph_build_cache_size` (int | None): Override graph build cache size (default: `None`, uses engine default).
 - `mutations_before_rebuild` (int | None): Override rebuild threshold (default: `None`, uses engine default).

bindings/python/docs/api/vector.md

@@ -155,6 +155,7 @@ db.create_vector_index(
     max_connections: int = 16,
     beam_width: int = 100,
     quantization: str = "INT8",
+    encoding: str | None = None,
     location_cache_size: int | None = None,
     graph_build_cache_size: int | None = None,
     mutations_before_rebuild: int | None = None,
@@ -194,6 +195,10 @@ db.create_vector_index(
       to a small value or prefer `"INT8"`, `"BINARY"`, or `None`.
     - Prefer `"INT8"` for current production usage in these bindings.
     - `"PRODUCT"`/PQ is available but currently not recommended for production workloads.
+- `encoding` (str | None): Optional storage encoding for the vector property.
+    - Use `"INT8"` with a `BINARY` property when your vectors are already stored as
+    signed bytes.
+    - Pair `encoding="INT8"` with `quantization="NONE"` to avoid double quantization.
 - `build_graph_now` (bool): If `True` (default), eagerly prepares the vector graph
   during index creation. Set to `False` to defer graph preparation until first query.
 
@@ -239,6 +244,21 @@ Treat this as a helper/manual API. For normal application queries, prefer SQL
 `vectorNeighbors` so search composes naturally with filtering, projection, and record
 exclusion.
 
+---
+
+### `to_java_byte_array(vector)`
+
+Convert a Python byte-like or integer array-like object to a Java `byte[]`.
+
+Use this when inserting native INT8 vectors into a `BINARY` property for indexes
+created with `encoding="INT8"`.
+
+```python
+from arcadedb_embedded import to_java_byte_array
+
+payload = to_java_byte_array([127, 0, -12, 5])
+```
+
 **Note:** With default settings (`build_graph_now=True` in `create_vector_index`), graph
 preparation runs during index creation. In the preferred SQL path, this eager behavior is
 also the default. If you explicitly disable eager preparation, the first call to

bindings/python/docs/guide/vectors.md

@@ -62,7 +62,9 @@ Preferred split:
 - Keep the secondary Python helper APIs in mind only for manual or maintenance cases;
   they are not the recommended application-facing workflow.
 
-- Vector property type must be `ARRAY_OF_FLOATS`.
+- Vector property type is usually `ARRAY_OF_FLOATS`.
+- Use `BINARY` only when you are storing pre-quantized INT8 bytes with
+  `encoding="INT8"`.
 - `CREATE INDEX ON Doc (embedding) LSM_VECTOR METADATA {...}` is the preferred creation
   path.
     - SQL builds the vector graph immediately by default.
@@ -241,6 +243,96 @@ rows = db.query(
 - Quantization via SQL: `METADATA {"quantization": "INT8"}` is the recommended path for
   embedded usage.
 
+## Native INT8 Storage
+
+If your application already has INT8 vectors, store them in a `BINARY` property and
+set `encoding="INT8"` on the vector index metadata.
+
+```python
+import arcadedb_embedded as arcadedb
+
+with arcadedb.create_database("./vector_demo_int8") as db:
+    db.command("sql", "CREATE VERTEX TYPE ByteDoc")
+    db.command("sql", "CREATE PROPERTY ByteDoc.id STRING")
+    db.command("sql", "CREATE PROPERTY ByteDoc.embedding BINARY")
+
+    db.command(
+    "sql",
+    """
+    CREATE INDEX ON ByteDoc (embedding)
+    LSM_VECTOR
+    METADATA {
+        "dimensions": 4,
+        "similarity": "COSINE",
+        "quantization": "NONE",
+        "encoding": "INT8"
+    }
+    """,
+    )
+
+    with db.transaction():
+    db.command(
+        "sql",
+        "INSERT INTO ByteDoc SET id = ?, embedding = ?",
+        "doc_a",
+        arcadedb.to_java_byte_array([127, 0, 0, 0]),
+    )
+```
+
+Use `encoding="INT8"` only with `quantization="NONE"`. Combining INT8 storage encoding
+with INT8 quantization would quantize the same vector twice.
+
+## Sparse Vectors
+
+ArcadeDB also supports sparse top-K retrieval through `LSM_SPARSE_VECTOR` and
+`vector.sparseNeighbors(...)`.
+
+```python
+import arcadedb_embedded as arcadedb
+import jpype.types as jtypes
+
+with arcadedb.create_database("./sparse_demo") as db:
+    db.command("sql", "CREATE DOCUMENT TYPE SparseDoc")
+    db.command("sql", "CREATE PROPERTY SparseDoc.tokens ARRAY_OF_INTEGERS")
+    db.command("sql", "CREATE PROPERTY SparseDoc.weights ARRAY_OF_FLOATS")
+
+    db.command(
+    "sql",
+    """
+    CREATE INDEX ON SparseDoc (tokens, weights)
+    LSM_SPARSE_VECTOR
+    METADATA {"dimensions": 128}
+    """,
+    )
+
+    rows = db.query(
+    "sql",
+    "SELECT expand(`vector.sparseNeighbors`('SparseDoc[tokens,weights]', ?, ?, 5))",
+    jtypes.JArray(jtypes.JInt)([5]),
+    arcadedb.to_java_float_array([1.0]),
+    ).to_list()
+```
+
+## Grouped Search
+
+Recent engine builds support `groupBy` / `groupSize` options on `vector.neighbors`.
+This is useful when you want diversity across a field such as source file, tenant, or
+document family.
+
+```python
+rows = db.query(
+    "sql",
+    (
+    "SELECT source_file, distance FROM "
+    "(SELECT expand(`vector.neighbors`(?, ?, ?, { groupBy: 'source_file', groupSize: 1 }))) "
+    "ORDER BY distance"
+    ),
+    "GroupedDoc[embedding]",
+    arcadedb.to_java_float_array([1.0, 0.0, 0.0, 0.0]),
+    3,
+).to_list()
+```
+
 ## Examples & References
 
 - **[Example 03: Vector Search – Semantic Similarity](../examples/03_vector_search.md)**

bindings/python/examples/03_vector_search.py

@@ -10,6 +10,8 @@
 - Creating HNSW (JVector) indexes for fast nearest-neighbor search
 - Finding semantically similar documents using cosine similarity
 - Understanding vector search parameters (dimensions, distance functions)
+- INT8-encoded dense-vector storage for smaller payloads and bucket footprint
+- Sparse-vector indexing for token/weight retrieval workloads
 - Index population strategies and performance characteristics
 
 Implementation Status:
@@ -45,6 +47,7 @@
 import time
 
 import arcadedb_embedded as arcadedb
+import jpype.types as jtypes
 import numpy as np
 
 # Parse command line arguments
@@ -139,6 +142,11 @@ def create_mock_embedding(category_seed, doc_seed):
 
         return embedding.astype(np.float32)
 
+    def quantize_to_int8_bytes(vector: np.ndarray):
+        """Quantize a normalized float vector to signed int8 bytes."""
+        scaled = np.clip(np.rint(vector * 127.0), -127, 127).astype(np.int8)
+        return scaled.tolist()
+
     # Generate documents
     documents = []
     for i in range(NUM_DOCUMENTS):
@@ -312,6 +320,129 @@ def create_mock_embedding(category_seed, doc_seed):
     print(f"   ⏱️  All queries time: {time.time() - step_start:.3f}s")
     print()
 
+    # -----------------------------------------------------------------------------
+    # Step 7: INT8-Encoded Dense Vectors
+    # -----------------------------------------------------------------------------
+    print("Step 7: Demonstrating INT8-encoded dense-vector storage...")
+    step_start = time.time()
+
+    try:
+        db.command("sql", "CREATE VERTEX TYPE Int8Article")
+        db.command("sql", "CREATE PROPERTY Int8Article.id STRING")
+        db.command("sql", "CREATE PROPERTY Int8Article.category STRING")
+        db.command("sql", "CREATE PROPERTY Int8Article.embedding BINARY")
+
+        db.command(
+            "sql",
+            """
+            CREATE INDEX ON Int8Article (embedding)
+            LSM_VECTOR
+            METADATA {
+                "dimensions": 4,
+                "similarity": "COSINE",
+                "quantization": "NONE",
+                "encoding": "INT8"
+            }
+            """,
+        )
+
+        int8_docs = [
+            ("int8_doc_1", "technology", [1.0, 0.0, 0.0, 0.0]),
+            ("int8_doc_2", "technology", [0.95, 0.05, 0.0, 0.0]),
+            ("int8_doc_3", "sports", [0.0, 1.0, 0.0, 0.0]),
+        ]
+
+        with db.transaction():
+            for doc_id, category, vector in int8_docs:
+                db.command(
+                    "sql",
+                    "INSERT INTO Int8Article SET id = ?, category = ?, embedding = ?",
+                    doc_id,
+                    category,
+                    arcadedb.to_java_byte_array(
+                        quantize_to_int8_bytes(np.array(vector))
+                    ),
+                )
+
+        int8_hits = db.query(
+            "sql",
+            (
+                "SELECT id, category, distance FROM "
+                "(SELECT expand(vectorNeighbors('Int8Article[embedding]', ?, 2))) "
+                "ORDER BY distance"
+            ),
+            arcadedb.to_java_float_array([1.0, 0.0, 0.0, 0.0]),
+        ).to_list()
+    except arcadedb.ArcadeDBError as exc:
+        print("   ⚠️  Skipping INT8-encoded dense-vector demo in this runtime")
+        print(f"   💡 Reason: {exc}")
+    else:
+        print("   ✅ Created INT8-encoded dense index on a BINARY property")
+        print("   💡 Use this when your embeddings are already stored as int8 bytes")
+        print("   Top matches for [1, 0, 0, 0]:")
+        for hit in int8_hits:
+            print(
+                f"      • {hit.get('id')} ({hit.get('category')}), "
+                f"distance={hit.get('distance'):.4f}"
+            )
+    print(f"   ⏱️  Time: {time.time() - step_start:.3f}s")
+    print()
+
+    # -----------------------------------------------------------------------------
+    # Step 8: Sparse Vectors
+    # -----------------------------------------------------------------------------
+    print("Step 8: Demonstrating sparse-vector retrieval...")
+    step_start = time.time()
+
+    try:
+        db.command("sql", "CREATE DOCUMENT TYPE SparseArticle")
+        db.command("sql", "CREATE PROPERTY SparseArticle.id STRING")
+        db.command("sql", "CREATE PROPERTY SparseArticle.tokens ARRAY_OF_INTEGERS")
+        db.command("sql", "CREATE PROPERTY SparseArticle.weights ARRAY_OF_FLOATS")
+
+        db.command(
+            "sql",
+            """
+            CREATE INDEX ON SparseArticle (tokens, weights)
+            LSM_SPARSE_VECTOR
+            METADATA {
+                "dimensions": 128
+            }
+            """,
+        )
+
+        with db.transaction():
+            db.command(
+                "sql",
+                "INSERT INTO SparseArticle SET id = 'sparse_doc_1', tokens = [1, 5, 10], weights = [0.5, 0.3, 0.2]",
+            )
+            db.command(
+                "sql",
+                "INSERT INTO SparseArticle SET id = 'sparse_doc_2', tokens = [2, 5, 11], weights = [0.4, 0.6, 0.1]",
+            )
+
+        sparse_hits = db.query(
+            "sql",
+            (
+                "SELECT id, score FROM "
+                "(SELECT expand(`vector.sparseNeighbors`('SparseArticle[tokens,weights]', ?, ?, 5))) "
+                "ORDER BY score DESC"
+            ),
+            jtypes.JArray(jtypes.JInt)([5]),
+            arcadedb.to_java_float_array([1.0]),
+        ).to_list()
+    except arcadedb.ArcadeDBError as exc:
+        print("   ⚠️  Skipping sparse-vector demo in this runtime")
+        print(f"   💡 Reason: {exc}")
+    else:
+        print("   ✅ Created sparse-vector index on token/weight arrays")
+        print("   💡 Use this for BM25-style or learned sparse retrieval")
+        print("   Top matches for sparse query {(5): 1.0}:")
+        for hit in sparse_hits:
+            print(f"      • {hit.get('id')}, score={hit.get('score'):.4f}")
+    print(f"   ⏱️  Time: {time.time() - step_start:.3f}s")
+    print()
+
     # -----------------------------------------------------------------------------
     # Cleanup
     # -----------------------------------------------------------------------------

bindings/python/src/arcadedb_embedded/__init__.py

@@ -58,7 +58,12 @@
 from .type_conversion import convert_java_to_python, convert_python_to_java
 
 # Import vector utilities and index
-from .vector import VectorIndex, to_java_float_array, to_python_array
+from .vector import (
+    VectorIndex,
+    to_java_byte_array,
+    to_java_float_array,
+    to_python_array,
+)
 
 __all__ = [
     "__version__",
@@ -91,6 +96,7 @@
     "convert_python_to_java",
     # Vector search
     "VectorIndex",
+    "to_java_byte_array",
     "to_java_float_array",
     "to_python_array",
     # Data export