Refactor build script and enhance vector handling in Python bindings

- Updated the build script to clarify build methods and improve architecture normalization.
- Added support for encoding in the Database class to handle INT8 storage.
- Introduced a new utility function to convert Python byte-like objects to Java byte arrays.
- Enhanced vector-related functions and tests to support new encoding and quantization features.
- Added tests for full-text search with wildcards and vector neighbors with group options.
- Improved error handling and documentation across various test cases.

Author: tae898

bindings/python/README.md

@@ -2,7 +2,7 @@
 
 Native Python bindings for ArcadeDB - the multi-model database that supports Graph, Document, Key/Value, Search Engine, Time Series, and Vector models.
 
-**Status**: ✅ Production Ready | **Tests**: 290 Passed | **Platforms**: 4 Supported
+**Status**: ✅ Production Ready | **Tests**: 331 Passed | **Platforms**: 4 Supported
 
 ---
 
@@ -92,7 +92,7 @@ Import: `import arcadedb_embedded as arcadedb`
 
 ## 🧪 Testing
 
-**Status**: 290 passed
+**Status**: 331 passed
 
 ```bash
 # Run all tests

bindings/python/examples/03_vector_search.py

@@ -10,6 +10,8 @@
 - Creating HNSW (JVector) indexes for fast nearest-neighbor search
 - Finding semantically similar documents using cosine similarity
 - Understanding vector search parameters (dimensions, distance functions)
+- INT8-encoded dense-vector storage for smaller payloads and bucket footprint
+- Sparse-vector indexing for token/weight retrieval workloads
 - Index population strategies and performance characteristics
 
 Implementation Status:
@@ -45,6 +47,7 @@
 import time
 
 import arcadedb_embedded as arcadedb
+import jpype.types as jtypes
 import numpy as np
 
 # Parse command line arguments
@@ -139,6 +142,11 @@ def create_mock_embedding(category_seed, doc_seed):
 
         return embedding.astype(np.float32)
 
+    def quantize_to_int8_bytes(vector: np.ndarray):
+        """Quantize a normalized float vector to signed int8 bytes."""
+        scaled = np.clip(np.rint(vector * 127.0), -127, 127).astype(np.int8)
+        return scaled.tolist()
+
     # Generate documents
     documents = []
     for i in range(NUM_DOCUMENTS):
@@ -312,6 +320,129 @@ def create_mock_embedding(category_seed, doc_seed):
     print(f"   ⏱️  All queries time: {time.time() - step_start:.3f}s")
     print()
 
+    # -----------------------------------------------------------------------------
+    # Step 7: INT8-Encoded Dense Vectors
+    # -----------------------------------------------------------------------------
+    print("Step 7: Demonstrating INT8-encoded dense-vector storage...")
+    step_start = time.time()
+
+    try:
+        db.command("sql", "CREATE VERTEX TYPE Int8Article")
+        db.command("sql", "CREATE PROPERTY Int8Article.id STRING")
+        db.command("sql", "CREATE PROPERTY Int8Article.category STRING")
+        db.command("sql", "CREATE PROPERTY Int8Article.embedding BINARY")
+
+        db.command(
+            "sql",
+            """
+            CREATE INDEX ON Int8Article (embedding)
+            LSM_VECTOR
+            METADATA {
+                "dimensions": 4,
+                "similarity": "COSINE",
+                "quantization": "NONE",
+                "encoding": "INT8"
+            }
+            """,
+        )
+
+        int8_docs = [
+            ("int8_doc_1", "technology", [1.0, 0.0, 0.0, 0.0]),
+            ("int8_doc_2", "technology", [0.95, 0.05, 0.0, 0.0]),
+            ("int8_doc_3", "sports", [0.0, 1.0, 0.0, 0.0]),
+        ]
+
+        with db.transaction():
+            for doc_id, category, vector in int8_docs:
+                db.command(
+                    "sql",
+                    "INSERT INTO Int8Article SET id = ?, category = ?, embedding = ?",
+                    doc_id,
+                    category,
+                    arcadedb.to_java_byte_array(
+                        quantize_to_int8_bytes(np.array(vector))
+                    ),
+                )
+
+        int8_hits = db.query(
+            "sql",
+            (
+                "SELECT id, category, distance FROM "
+                "(SELECT expand(vectorNeighbors('Int8Article[embedding]', ?, 2))) "
+                "ORDER BY distance"
+            ),
+            arcadedb.to_java_float_array([1.0, 0.0, 0.0, 0.0]),
+        ).to_list()
+    except arcadedb.ArcadeDBError as exc:
+        print("   ⚠️  Skipping INT8-encoded dense-vector demo in this runtime")
+        print(f"   💡 Reason: {exc}")
+    else:
+        print("   ✅ Created INT8-encoded dense index on a BINARY property")
+        print("   💡 Use this when your embeddings are already stored as int8 bytes")
+        print("   Top matches for [1, 0, 0, 0]:")
+        for hit in int8_hits:
+            print(
+                f"      • {hit.get('id')} ({hit.get('category')}), "
+                f"distance={hit.get('distance'):.4f}"
+            )
+    print(f"   ⏱️  Time: {time.time() - step_start:.3f}s")
+    print()
+
+    # -----------------------------------------------------------------------------
+    # Step 8: Sparse Vectors
+    # -----------------------------------------------------------------------------
+    print("Step 8: Demonstrating sparse-vector retrieval...")
+    step_start = time.time()
+
+    try:
+        db.command("sql", "CREATE DOCUMENT TYPE SparseArticle")
+        db.command("sql", "CREATE PROPERTY SparseArticle.id STRING")
+        db.command("sql", "CREATE PROPERTY SparseArticle.tokens ARRAY_OF_INTEGERS")
+        db.command("sql", "CREATE PROPERTY SparseArticle.weights ARRAY_OF_FLOATS")
+
+        db.command(
+            "sql",
+            """
+            CREATE INDEX ON SparseArticle (tokens, weights)
+            LSM_SPARSE_VECTOR
+            METADATA {
+                "dimensions": 128
+            }
+            """,
+        )
+
+        with db.transaction():
+            db.command(
+                "sql",
+                "INSERT INTO SparseArticle SET id = 'sparse_doc_1', tokens = [1, 5, 10], weights = [0.5, 0.3, 0.2]",
+            )
+            db.command(
+                "sql",
+                "INSERT INTO SparseArticle SET id = 'sparse_doc_2', tokens = [2, 5, 11], weights = [0.4, 0.6, 0.1]",
+            )
+
+        sparse_hits = db.query(
+            "sql",
+            (
+                "SELECT id, score FROM "
+                "(SELECT expand(`vector.sparseNeighbors`('SparseArticle[tokens,weights]', ?, ?, 5))) "
+                "ORDER BY score DESC"
+            ),
+            jtypes.JArray(jtypes.JInt)([5]),
+            arcadedb.to_java_float_array([1.0]),
+        ).to_list()
+    except arcadedb.ArcadeDBError as exc:
+        print("   ⚠️  Skipping sparse-vector demo in this runtime")
+        print(f"   💡 Reason: {exc}")
+    else:
+        print("   ✅ Created sparse-vector index on token/weight arrays")
+        print("   💡 Use this for BM25-style or learned sparse retrieval")
+        print("   Top matches for sparse query {(5): 1.0}:")
+        for hit in sparse_hits:
+            print(f"      • {hit.get('id')}, score={hit.get('score'):.4f}")
+    print(f"   ⏱️  Time: {time.time() - step_start:.3f}s")
+    print()
+
     # -----------------------------------------------------------------------------
     # Cleanup
     # -----------------------------------------------------------------------------

bindings/python/examples/11_vector_index_build.py

@@ -98,7 +98,7 @@ def fetch_json(url: str) -> dict:
     if not url.startswith("https://"):
         raise ValueError(f"Refusing to open non-HTTPS URL: {url!r}")
     req = Request(url, headers={"User-Agent": "arcadedb-bench"})
-    with urlopen(req, timeout=30) as response:  # nosec B310 - https-only
+    with urlopen(req, timeout=30) as response:  # nosec B310
         payload = json.load(response)
     if not isinstance(payload, dict):
         raise RuntimeError(f"Expected JSON object from {url}")
@@ -495,24 +495,46 @@ def stream_shards(
         remaining -= take_total
 
 
+def quantize_to_int8_bytes(vector: np.ndarray) -> list[int]:
+    array = np.asarray(vector, dtype=np.float32)
+    if not np.all(np.isfinite(array)):
+        raise ValueError("Cannot INT8-encode vectors with NaN or infinite values")
+    scaled = np.clip(np.rint(array * 127.0), -127, 127).astype(np.int8)
+    return scaled.tolist()
+
+
 def ingest_vectors_arcadedb(
     db,
     sources: List[dict],
     dim: int,
     count: int,
     batch_size: int,
     to_java_float_array,
+    to_java_byte_array,
+    encoding: str,
 ) -> int:
+    use_int8_encoding = encoding.upper() == "INT8"
+
     for command in (
         "CREATE VERTEX TYPE VectorData",
         "CREATE PROPERTY VectorData.id INTEGER",
-        "CREATE PROPERTY VectorData.vector ARRAY_OF_FLOATS",
+        (
+            "CREATE PROPERTY VectorData.vector BINARY"
+            if use_int8_encoding
+            else "CREATE PROPERTY VectorData.vector ARRAY_OF_FLOATS"
+        ),
     ):
         try:
             db.command("sql", command)
         except Exception:
             pass
 
+    if use_int8_encoding and to_java_byte_array is None:
+        raise RuntimeError(
+            "arcadedb_embedded does not expose to_java_byte_array; update the "
+            "local wheel before using --encoding INT8"
+        )
+
     ingested = 0
     for base_id, batch in stream_shards(
         sources,
@@ -522,7 +544,14 @@ def ingest_vectors_arcadedb(
     ):
         with db.transaction():
             for idx, vec in enumerate(batch, start=base_id):
-                jvec = to_java_float_array(vec) if to_java_float_array else vec.tolist()
+                if use_int8_encoding:
+                    jvec = to_java_byte_array(quantize_to_int8_bytes(vec))
+                else:
+                    jvec = (
+                        to_java_float_array(vec)
+                        if to_java_float_array
+                        else vec.tolist()
+                    )
                 db.command(
                     "sql",
                     "INSERT INTO VectorData SET id = ?, vector = ?",
@@ -540,10 +569,19 @@ def create_index_arcadedb(
     max_connections: int,
     beam_width: int,
     quantization: str,
+    encoding: str,
     store_vectors_in_graph: bool,
     add_hierarchy: bool,
 ):
     quant = None if quantization.upper() == "NONE" else quantization.upper()
+    enc = None if encoding.upper() == "NONE" else encoding.upper()
+
+    if enc == "INT8" and quant == "INT8":
+        raise ValueError(
+            "--encoding INT8 cannot be combined with --quantization INT8; "
+            "use --quantization NONE for native INT8 storage"
+        )
+
     metadata_lines = [
         f'"dimensions": {int(dim)}',
         '"similarity": "COSINE"',
@@ -554,6 +592,8 @@ def create_index_arcadedb(
     ]
     if quant is not None:
         metadata_lines.insert(4, f'"quantization": "{quant}"')
+    if enc is not None:
+        metadata_lines.insert(5, f'"encoding": "{enc}"')
 
     metadata_body = ",\n            ".join(metadata_lines)
 
@@ -964,9 +1004,7 @@ def wait_for_qdrant_ready(host: str, port: int, timeout_sec: int = 120) -> None:
     while True:
         for url in urls:
             try:
-                with urlopen(
-                    url, timeout=3
-                ) as response:  # nosec B310 - localhost health-check URL
+                with urlopen(url, timeout=3) as response:  # nosec B310
                     if 200 <= int(response.status) < 500:
                         return
             except Exception:
@@ -1016,9 +1054,7 @@ def ensure_milvus_compose_file(compose_file: Path, release_tag: str) -> None:
             "https://github.com/milvus-io/milvus/releases/download/"
             f"{release_tag}/milvus-standalone-docker-compose.yml"
         )
-        urlretrieve(
-            url, str(compose_file)
-        )  # nosec B310 - url is a hardcoded https://github.com URL
+        urlretrieve(url, str(compose_file))  # nosec B310
         raw = compose_file.read_text(encoding="utf-8")
 
     sanitized = re.sub(r"(?m)^\s*container_name:\s*.*\n", "", raw)
@@ -1863,6 +1899,12 @@ def main() -> None:
         default="NONE",
         help="ArcadeDB quantization mode (ignored by pgvector)",
     )
+    parser.add_argument(
+        "--encoding",
+        choices=["NONE", "INT8"],
+        default="NONE",
+        help="ArcadeDB storage encoding for the vector property (default: NONE)",
+    )
     parser.add_argument(
         "--store-vectors-in-graph",
         action="store_true",
@@ -1958,6 +2000,12 @@ def main() -> None:
         args.milvus_compose_version = resolve_milvus_compose_version(
             args.milvus_compose_version
         )
+    if args.encoding != "NONE" and args.backend != "arcadedb_sql":
+        parser.error("--encoding is supported only for backend=arcadedb_sql")
+    if args.encoding == "INT8" and args.quantization != "NONE":
+        parser.error(
+            "--encoding INT8 requires --quantization NONE to avoid double quantization"
+        )
     if args.backend == "pgvector" and args.docker_image == "python:3.12-slim":
         args.docker_image = resolve_latest_pgvector_image()
     configure_reproducibility(args.seed)
@@ -2001,6 +2049,7 @@ def main() -> None:
             f"maxconn={args.max_connections}",
             f"beam={args.beam_width}",
             f"quant={args.quantization.lower()}",
+            f"enc={args.encoding.lower()}",
             f"store={'on' if args.store_vectors_in_graph else 'off'}",
             f"hier={'on' if args.add_hierarchy else 'off'}",
             f"batch={args.batch_size}",
@@ -2028,6 +2077,8 @@ def main() -> None:
     print(f"Dimensions: {dim}")
     print(f"Max connections: {args.max_connections}")
     print(f"Beam width: {args.beam_width}")
+    print(f"Quantization: {args.quantization}")
+    print(f"Encoding: {args.encoding}")
     if args.backend == "pgvector":
         print(f"Postgres shared_buffers: {pg_shared_buffers}")
     print(f"DB path: {db_path}")
@@ -2067,6 +2118,7 @@ def record(name: str, result, dur: float, rss_start: float, rss_end: float):
 
         try:
             to_java_float_array = getattr(arcadedb, "to_java_float_array", None)
+            to_java_byte_array = getattr(arcadedb, "to_java_byte_array", None)
             ingest_started_at = datetime.now(timezone.utc).isoformat()
             print(f"Ingest start (arcadedb, UTC): {ingest_started_at}")
             ingested, dur, r0, r1 = timed_section(
@@ -2078,6 +2130,8 @@ def record(name: str, result, dur: float, rss_start: float, rss_end: float):
                     count=count,
                     batch_size=args.batch_size,
                     to_java_float_array=to_java_float_array,
+                    to_java_byte_array=to_java_byte_array,
+                    encoding=args.encoding,
                 ),
             )
             ingest_ended_at = datetime.now(timezone.utc).isoformat()
@@ -2095,6 +2149,7 @@ def record(name: str, result, dur: float, rss_start: float, rss_end: float):
                     max_connections=args.max_connections,
                     beam_width=args.beam_width,
                     quantization=args.quantization,
+                    encoding=args.encoding,
                     store_vectors_in_graph=args.store_vectors_in_graph,
                     add_hierarchy=args.add_hierarchy,
                 ),
@@ -2557,6 +2612,7 @@ def server_pid_provider() -> int | None:
             "max_connections": args.max_connections,
             "beam_width": args.beam_width,
             "quantization": args.quantization,
+            "encoding": args.encoding,
             "store_vectors_in_graph": args.store_vectors_in_graph,
             "add_hierarchy": args.add_hierarchy,
             "batch_size": args.batch_size,