From ebab58cf496979bec6f7b12c5eff9d4e3fe279ff Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sat, 19 Jul 2025 15:53:02 -0700 Subject: [PATCH 01/10] fix: update FtsQuery to use properties pattern instead of oneOf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Changed FtsQuery from oneOf pattern to properties-based pattern - Updated BoostQuery to reference FtsQuery for positive/negative fields - Modified QueryRequest full_text_query to support both string and structured queries - Updated Java tests to use new QueryRequestFullTextQuery wrapper - Added documentation about oneOf pattern handling in CLAUDE.md This change improves compatibility across different code generators (Java, Python, Rust) and follows the same pattern as AlterTransactionAction. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- CLAUDE.md | 135 ++++++++ docs/src/spec/rest.yaml | 67 ++-- java/LANCEDB_VECTOR_QUERY_ANALYSIS.md | 140 ++++++++ java/lance-namespace-apache-client/README.md | 6 +- .../api/openapi.yaml | 313 +++++++++++++++--- .../docs/BoostQuery.md | 7 +- .../docs/FtsQuery.md | 11 +- .../docs/FtsQueryOneOf.md | 13 - .../docs/FtsQueryOneOf1.md | 13 - .../docs/FtsQueryOneOf2.md | 13 - .../docs/FtsQueryOneOf3.md | 13 - .../docs/FtsQueryOneOf4.md | 13 - .../docs/QueryRequest.md | 2 +- .../docs/QueryRequestFullTextQuery.md | 15 + .../lance/namespace/model/BoostQuery.java | 116 +++---- .../lance/namespace/model/FtsQuery.java | 66 ++-- .../lance/namespace/model/FtsQueryOneOf.java | 135 -------- .../lance/namespace/model/FtsQueryOneOf1.java | 135 -------- .../lance/namespace/model/FtsQueryOneOf2.java | 135 -------- .../lance/namespace/model/FtsQueryOneOf3.java | 135 -------- .../lance/namespace/model/FtsQueryOneOf4.java | 135 -------- .../lance/namespace/model/QueryRequest.java | 11 +- .../model/QueryRequestFullTextQuery.java | 174 ++++++++++ .../lance/namespace/RestTableApiTest.java | 18 +- .../server/springboot/model/BooleanQuery.java | 27 +- .../server/springboot/model/BoostQuery.java | 82 +++-- .../server/springboot/model/FtsQuery.java | 176 +++++++++- .../springboot/model/FtsQueryOneOf.java | 102 ------ .../springboot/model/FtsQueryOneOf1.java | 102 ------ .../springboot/model/FtsQueryOneOf2.java | 102 ------ .../springboot/model/FtsQueryOneOf3.java | 102 ------ .../springboot/model/FtsQueryOneOf4.java | 102 ------ .../server/springboot/model/QueryRequest.java | 15 +- .../model/QueryRequestFullTextQuery.java | 121 +++++++ .../lance_namespace_urllib3_client/README.md | 6 +- .../docs/BoostQuery.md | 7 +- .../docs/FtsQuery.md | 11 +- .../docs/FtsQueryOneOf.md | 29 -- .../docs/FtsQueryOneOf1.md | 29 -- .../docs/FtsQueryOneOf2.md | 29 -- .../docs/FtsQueryOneOf3.md | 29 -- .../docs/FtsQueryOneOf4.md | 29 -- .../docs/QueryRequest.md | 2 +- .../docs/QueryRequestFullTextQuery.md | 31 ++ .../__init__.py | 6 +- .../models/__init__.py | 6 +- .../models/boost_query.py | 27 +- .../models/fts_query.py | 224 +++++-------- .../models/fts_query_one_of1.py | 91 ----- .../models/fts_query_one_of2.py | 91 ----- .../models/fts_query_one_of3.py | 91 ----- .../models/fts_query_one_of4.py | 93 ------ .../models/query_request.py | 6 +- ...of.py => query_request_full_text_query.py} | 30 +- .../test/test_boolean_query.py | 294 +++++++++++++++- .../test/test_boost_query.py | 206 +++++++++++- .../test/test_fts_query.py | 106 +++--- .../test/test_fts_query_one_of.py | 66 ---- .../test/test_fts_query_one_of1.py | 58 ---- .../test/test_fts_query_one_of2.py | 58 ---- .../test/test_fts_query_one_of3.py | 72 ---- .../test/test_fts_query_one_of4.py | 70 ---- .../test/test_query_request.py | 62 +++- .../test_query_request_full_text_query.py | 106 ++++++ .../test/test_structured_fts_query.py | 102 +++++- rust/lance-namespace-reqwest-client/README.md | 6 +- .../docs/BoostQuery.md | 6 +- .../docs/FtsQuery.md | 18 +- .../docs/FtsQueryOneOf.md | 11 - .../docs/FtsQueryOneOf1.md | 11 - .../docs/FtsQueryOneOf2.md | 11 - .../docs/FtsQueryOneOf4.md | 11 - .../docs/QueryRequest.md | 2 +- ...OneOf3.md => QueryRequestFullTextQuery.md} | 5 +- .../src/models/boost_query.rs | 15 +- .../src/models/fts_query.rs | 34 +- .../src/models/fts_query_one_of.rs | 27 -- .../src/models/fts_query_one_of_1.rs | 27 -- .../src/models/fts_query_one_of_2.rs | 27 -- .../src/models/fts_query_one_of_4.rs | 27 -- .../src/models/mod.rs | 12 +- .../src/models/query_request.rs | 3 +- ..._3.rs => query_request_full_text_query.rs} | 19 +- 83 files changed, 2279 insertions(+), 2812 deletions(-) create mode 100644 CLAUDE.md create mode 100644 java/LANCEDB_VECTOR_QUERY_ANALYSIS.md delete mode 100644 java/lance-namespace-apache-client/docs/FtsQueryOneOf.md delete mode 100644 java/lance-namespace-apache-client/docs/FtsQueryOneOf1.md delete mode 100644 java/lance-namespace-apache-client/docs/FtsQueryOneOf2.md delete mode 100644 java/lance-namespace-apache-client/docs/FtsQueryOneOf3.md delete mode 100644 java/lance-namespace-apache-client/docs/FtsQueryOneOf4.md create mode 100644 java/lance-namespace-apache-client/docs/QueryRequestFullTextQuery.md delete mode 100644 java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf.java delete mode 100644 java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf1.java delete mode 100644 java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf2.java delete mode 100644 java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf3.java delete mode 100644 java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf4.java create mode 100644 java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequestFullTextQuery.java delete mode 100644 java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf.java delete mode 100644 java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf1.java delete mode 100644 java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf2.java delete mode 100644 java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf3.java delete mode 100644 java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf4.java create mode 100644 java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequestFullTextQuery.java delete mode 100644 python/lance_namespace_urllib3_client/docs/FtsQueryOneOf.md delete mode 100644 python/lance_namespace_urllib3_client/docs/FtsQueryOneOf1.md delete mode 100644 python/lance_namespace_urllib3_client/docs/FtsQueryOneOf2.md delete mode 100644 python/lance_namespace_urllib3_client/docs/FtsQueryOneOf3.md delete mode 100644 python/lance_namespace_urllib3_client/docs/FtsQueryOneOf4.md create mode 100644 python/lance_namespace_urllib3_client/docs/QueryRequestFullTextQuery.md delete mode 100644 python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of1.py delete mode 100644 python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of2.py delete mode 100644 python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of3.py delete mode 100644 python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of4.py rename python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/{fts_query_one_of.py => query_request_full_text_query.py} (67%) delete mode 100644 python/lance_namespace_urllib3_client/test/test_fts_query_one_of.py delete mode 100644 python/lance_namespace_urllib3_client/test/test_fts_query_one_of1.py delete mode 100644 python/lance_namespace_urllib3_client/test/test_fts_query_one_of2.py delete mode 100644 python/lance_namespace_urllib3_client/test/test_fts_query_one_of3.py delete mode 100644 python/lance_namespace_urllib3_client/test/test_fts_query_one_of4.py create mode 100644 python/lance_namespace_urllib3_client/test/test_query_request_full_text_query.py delete mode 100644 rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf.md delete mode 100644 rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf1.md delete mode 100644 rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf2.md delete mode 100644 rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf4.md rename rust/lance-namespace-reqwest-client/docs/{FtsQueryOneOf3.md => QueryRequestFullTextQuery.md} (55%) delete mode 100644 rust/lance-namespace-reqwest-client/src/models/fts_query_one_of.rs delete mode 100644 rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_1.rs delete mode 100644 rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_2.rs delete mode 100644 rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_4.rs rename rust/lance-namespace-reqwest-client/src/models/{fts_query_one_of_3.rs => query_request_full_text_query.rs} (55%) diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..f6903d467 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,135 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## OpenAPI oneOf Pattern Handling + +When working with OpenAPI specifications in this project, avoid using `oneOf` patterns as they cause compatibility issues across different language code generators. Instead, use the properties-based pattern similar to `AlterTransactionAction`: + +### Don't use oneOf: +```yaml +FtsQuery: + oneOf: + - type: object + required: [match] + properties: + match: + $ref: '#/components/schemas/MatchQuery' + - type: object + required: [phrase] + properties: + phrase: + $ref: '#/components/schemas/PhraseQuery' +``` + +### Do use properties pattern: +```yaml +FtsQuery: + type: object + description: | + Full-text search query. Exactly one query type field must be provided. + properties: + match: + $ref: '#/components/schemas/MatchQuery' + phrase: + $ref: '#/components/schemas/PhraseQuery' + boost: + $ref: '#/components/schemas/BoostQuery' +``` + +This pattern ensures better compatibility across different code generators (Java, Python, Rust) and maintains consistency with other parts of the API specification. + +## Project Overview + +Lance Namespace is an open specification for standardizing access to collections of Lance tables. The project provides: + +- An OpenAPI specification for REST-based namespace operations +- Multi-language client and server implementations (Java, Python, Rust) +- Generated code from the OpenAPI specification +- Documentation and examples + +## Architecture + +The project follows a specification-driven approach: + +1. **Core Specification**: OpenAPI spec at `docs/src/spec/rest.yaml` defines all operations +2. **Generated Code**: Client and server code is generated from the spec using `openapi-generator-cli` +3. **Multi-language Support**: Java, Python, and Rust implementations are provided +4. **Documentation**: MkDocs-based documentation in `docs/` directory + +### Key Components + +- **Java Core**: `java/lance-namespace-core/` - Core Java interface and utilities +- **Java Adapter**: `java/lance-namespace-adapter/` - Spring Boot server adapter +- **Generated Clients**: Auto-generated clients for each language +- **Generated Servers**: Auto-generated Spring Boot server stub + +## Development Commands + +### Project-wide Commands (from root) +```bash +# Lint the OpenAPI specification +make lint + +# Generate all clients and servers +make gen + +# Build all components +make build + +# Clean all generated code +make clean +``` + +### Language-specific Commands + +#### Java (from `java/` directory) +```bash +# Generate Java clients and servers +make gen + +# Build with Maven (includes linting with Spotless) +make build + +# Build specific components +make build-java-core +make build-java-adapter +``` + +#### Python (from `python/` directory) +```bash +# Generate Python client +make gen + +# Build and test with Poetry +make build +``` + +#### Rust (from `rust/` directory) +```bash +# Generate Rust client +make gen + +# Build and test with Cargo +make build +``` + +## Code Generation Workflow + +1. All client and server code is generated from `docs/src/spec/rest.yaml` +2. Generated code is placed in respective language directories +3. Build process includes generation, linting, and testing +4. Some generated files are cleaned up post-generation (metadata, git files) + +## Testing + +- **Java**: Tests run via Maven (`./mvnw install`) +- **Python**: Tests run via Poetry (`poetry run pytest`) +- **Rust**: Tests run via Cargo (`cargo test`) + +## Key Files + +- `docs/src/spec/rest.yaml`: OpenAPI specification (source of truth) +- `java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceNamespace.java`: Core Java interface +- `java/lance-namespace-adapter/`: Spring Boot server implementation +- Language-specific Makefiles for build automation \ No newline at end of file diff --git a/docs/src/spec/rest.yaml b/docs/src/spec/rest.yaml index 2ed30fb34..21c8cbb69 100644 --- a/docs/src/spec/rest.yaml +++ b/docs/src/spec/rest.yaml @@ -1502,6 +1502,7 @@ components: description: Whether to bypass vector index columns: type: array + nullable: true items: type: string description: Optional list of columns to return @@ -1527,8 +1528,14 @@ components: - 'null' description: Optional SQL filter expression full_text_query: - $ref: '#/components/schemas/StringFtsQuery' - description: Optional full-text search query (only string query supported) + type: object + nullable: true + description: Optional full-text search query. Provide either string_query or structured_query, not both. + properties: + string_query: + $ref: '#/components/schemas/StringFtsQuery' + structured_query: + $ref: '#/components/schemas/StructuredFtsQuery' k: type: integer minimum: 0 @@ -2107,37 +2114,22 @@ components: $ref: '#/components/schemas/FtsQuery' FtsQuery: - oneOf: - - type: object - required: - - match - properties: - match: - $ref: '#/components/schemas/MatchQuery' - - type: object - required: - - phrase - properties: - phrase: - $ref: '#/components/schemas/PhraseQuery' - - type: object - required: - - boost - properties: - boost: - $ref: '#/components/schemas/BoostQuery' - - type: object - required: - - multi_match - properties: - multi_match: - $ref: '#/components/schemas/MultiMatchQuery' - - type: object - required: - - boolean - properties: - boolean: - $ref: '#/components/schemas/BooleanQuery' + type: object + description: | + Full-text search query. Exactly one query type field must be provided. + This structure follows the same pattern as AlterTransactionAction to minimize + differences and compatibility issues across codegen in different languages. + properties: + match: + $ref: '#/components/schemas/MatchQuery' + phrase: + $ref: '#/components/schemas/PhraseQuery' + boost: + $ref: '#/components/schemas/BoostQuery' + multi_match: + $ref: '#/components/schemas/MultiMatchQuery' + boolean: + $ref: '#/components/schemas/BooleanQuery' MatchQuery: type: object @@ -2198,17 +2190,20 @@ components: BoostQuery: type: object + description: Boost query that scores documents matching positive query higher and negative query lower required: - positive - negative properties: + positive: + $ref: '#/components/schemas/FtsQuery' negative: - type: object + $ref: '#/components/schemas/FtsQuery' negative_boost: type: number format: float - positive: - type: object + description: 'Boost factor for negative query (default: 0.5)' + default: 0.5 MultiMatchQuery: type: object diff --git a/java/LANCEDB_VECTOR_QUERY_ANALYSIS.md b/java/LANCEDB_VECTOR_QUERY_ANALYSIS.md new file mode 100644 index 000000000..e2d2cd5e5 --- /dev/null +++ b/java/LANCEDB_VECTOR_QUERY_ANALYSIS.md @@ -0,0 +1,140 @@ +# LanceDB Rust Vector Query Implementation Analysis + +## Overview + +This document analyzes how LanceDB's Rust implementation handles vector query requests, focusing on the QueryVector handling and oneOf type patterns. + +## Key Components + +### 1. Query Structure Hierarchy + +The Rust implementation uses an enum pattern for different query types: + +```rust +pub enum AnyQuery { + Query(QueryRequest), // Regular queries without vector search + VectorQuery(VectorQueryRequest), // Vector-based queries +} +``` + +### 2. VectorQueryRequest Structure + +```rust +pub struct VectorQueryRequest { + pub base: QueryRequest, // Base query parameters + pub column: Option, // Target vector column + pub query_vector: Vec>, // Query vectors (supports multiple) + pub minimum_nprobes: usize, + pub maximum_nprobes: Option, + pub lower_bound: Option, + pub upper_bound: Option, + pub distance_type: Option, + pub ef: Option, + pub refine_factor: Option, + pub use_index: bool, +} +``` + +### 3. Vector Handling + +#### Input Types +- Vectors are stored as `Vec>` allowing multiple query vectors +- The implementation supports various input types through the `IntoQueryVector` trait +- Common conversions: `Vec`, `&[f32]`, `&[f16]`, Arrow arrays + +#### JSON Serialization +When sending to the server, vectors are converted to JSON: + +```rust +fn vector_to_json(vector: &arrow_array::ArrayRef) -> Result { + match vector.data_type() { + DataType::Float32 => { + let array = vector.as_any().downcast_ref::().unwrap(); + Ok(serde_json::Value::Array( + array.values().iter() + .map(|v| serde_json::Value::Number( + serde_json::Number::from_f64(*v as f64).unwrap() + )) + .collect(), + )) + } + _ => Err(Error::InvalidInput { + message: "VectorQuery vector must be of type Float32".into(), + }), + } +} +``` + +### 4. Query Building Pattern + +The Rust implementation uses a builder pattern: + +```rust +table.query() + .nearest_to(&[1.0, 2.0, 3.0]) // Sets the first query vector + .add_query_vector(&[4.0, 5.0, 6.0]) // Adds additional vectors + .column("my_vector_column") + .distance_type(DistanceType::Cosine) + .limit(10) + .execute() +``` + +### 5. Server Request Format + +The implementation prepares different request formats based on the number of query vectors: + +1. **No vectors**: `"vector": []` +2. **Single vector**: `"vector": [1.0, 2.0, 3.0]` +3. **Multiple vectors** (if server supports): + - New format: `"vector": [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]` + - Old format: Sends multiple separate requests + +### 6. Key Insights for Java Implementation + +1. **OneOf Pattern**: The Rust code uses enum for Query/VectorQuery distinction, which maps to OpenAPI oneOf +2. **Vector Format**: Always Float32 arrays serialized as JSON number arrays +3. **Multiple Vectors**: Support for batch vector queries with query_index in results +4. **Builder Pattern**: Clean API for constructing complex queries +5. **Server Version Handling**: Different behavior based on server capabilities + +### 7. Request Body Structure + +A typical vector query request body: + +```json +{ + "version": null, // or specific version number + "vector": [0.1, 0.2, 0.3], + "vector_column": "embeddings", + "distance_type": "cosine", + "k": 10, + "nprobes": 20, + "minimum_nprobes": 20, + "maximum_nprobes": 0, + "lower_bound": null, + "upper_bound": null, + "ef": null, + "refine_factor": null, + "bypass_vector_index": false, + "prefilter": true, + "filter": "id > 100", + "columns": ["id", "text", "metadata"] +} +``` + +### 8. Important Implementation Details + +1. **Type Safety**: Vectors must be Float32 type +2. **Empty Vector Handling**: Empty array `[]` means no vector search +3. **Column Auto-detection**: If column is not specified, server auto-detects +4. **Backward Compatibility**: Handles both old (nprobes) and new (minimum/maximum_nprobes) formats +5. **Timeout Support**: Query execution supports timeouts via headers + +## Recommendations for Java Implementation + +1. Use a similar builder pattern for query construction +2. Implement proper type conversion for vectors (List -> JSON array) +3. Handle the oneOf pattern with proper query type discrimination +4. Support both single and multiple vector queries +5. Implement server version detection for feature compatibility +6. Ensure proper JSON serialization matching the expected format \ No newline at end of file diff --git a/java/lance-namespace-apache-client/README.md b/java/lance-namespace-apache-client/README.md index 622f67f44..faafaf33c 100644 --- a/java/lance-namespace-apache-client/README.md +++ b/java/lance-namespace-apache-client/README.md @@ -174,11 +174,6 @@ Class | Method | HTTP request | Description - [DropTableResponse](docs/DropTableResponse.md) - [ErrorResponse](docs/ErrorResponse.md) - [FtsQuery](docs/FtsQuery.md) - - [FtsQueryOneOf](docs/FtsQueryOneOf.md) - - [FtsQueryOneOf1](docs/FtsQueryOneOf1.md) - - [FtsQueryOneOf2](docs/FtsQueryOneOf2.md) - - [FtsQueryOneOf3](docs/FtsQueryOneOf3.md) - - [FtsQueryOneOf4](docs/FtsQueryOneOf4.md) - [IndexListItemResponse](docs/IndexListItemResponse.md) - [IndexListRequest](docs/IndexListRequest.md) - [IndexListResponse](docs/IndexListResponse.md) @@ -201,6 +196,7 @@ Class | Method | HTTP request | Description - [Operator](docs/Operator.md) - [PhraseQuery](docs/PhraseQuery.md) - [QueryRequest](docs/QueryRequest.md) + - [QueryRequestFullTextQuery](docs/QueryRequestFullTextQuery.md) - [RegisterTableRequest](docs/RegisterTableRequest.md) - [RegisterTableResponse](docs/RegisterTableResponse.md) - [SetPropertyMode](docs/SetPropertyMode.md) diff --git a/java/lance-namespace-apache-client/api/openapi.yaml b/java/lance-namespace-apache-client/api/openapi.yaml index 0bbdbd7a4..e8cd12ce8 100644 --- a/java/lance-namespace-apache-client/api/openapi.yaml +++ b/java/lance-namespace-apache-client/api/openapi.yaml @@ -2022,19 +2022,64 @@ components: vector_column: vector_column fast_search: true k: 0 - upper_bound: 7.0614014 + upper_bound: 1.2315135 version: 0 with_row_id: true prefilter: true filter: filter refine_factor: 0 full_text_query: - columns: - - columns - - columns - query: query + string_query: + columns: + - columns + - columns + query: query + structured_query: + query: + boolean: + must_not: + - null + - null + should: + - null + - null + must: + - null + - null + phrase: + terms: terms + column: column + slop: 0 + match: + fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + boost: + negative: null + negative_boost: 7.0614014 + positive: null + multi_match: + match_queries: + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 distance_type: distance_type - lower_bound: 1.4658129 + lower_bound: 3.6160767 bypass_vector_index: true nprobes: 0 name: name @@ -2042,8 +2087,8 @@ components: - namespace - namespace vector: - - 9.301444 - - 9.301444 + - 1.0246457 + - 1.0246457 properties: name: type: string @@ -2060,6 +2105,7 @@ components: items: type: string type: array + nullable: true distance_type: description: Distance metric to use nullable: true @@ -2078,7 +2124,7 @@ components: nullable: true type: string full_text_query: - $ref: '#/components/schemas/StringFtsQuery' + $ref: '#/components/schemas/QueryRequest_full_text_query' k: description: Number of results to return minimum: 0 @@ -2777,19 +2823,123 @@ components: required: - query StructuredFtsQuery: + example: + query: + boolean: + must_not: + - null + - null + should: + - null + - null + must: + - null + - null + phrase: + terms: terms + column: column + slop: 0 + match: + fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + boost: + negative: null + negative_boost: 7.0614014 + positive: null + multi_match: + match_queries: + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 properties: query: $ref: '#/components/schemas/FtsQuery' required: - query FtsQuery: - oneOf: - - $ref: '#/components/schemas/FtsQuery_oneOf' - - $ref: '#/components/schemas/FtsQuery_oneOf_1' - - $ref: '#/components/schemas/FtsQuery_oneOf_2' - - $ref: '#/components/schemas/FtsQuery_oneOf_3' - - $ref: '#/components/schemas/FtsQuery_oneOf_4' + description: | + Full-text search query. Exactly one query type field must be provided. + This structure follows the same pattern as AlterTransactionAction to minimize + differences and compatibility issues across codegen in different languages. + example: + boolean: + must_not: + - null + - null + should: + - null + - null + must: + - null + - null + phrase: + terms: terms + column: column + slop: 0 + match: + fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + boost: + negative: null + negative_boost: 7.0614014 + positive: null + multi_match: + match_queries: + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + properties: + match: + $ref: '#/components/schemas/MatchQuery' + phrase: + $ref: '#/components/schemas/PhraseQuery' + boost: + $ref: '#/components/schemas/BoostQuery' + multi_match: + $ref: '#/components/schemas/MultiMatchQuery' + boolean: + $ref: '#/components/schemas/BooleanQuery' MatchQuery: + example: + fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 properties: boost: format: float @@ -2822,6 +2972,10 @@ components: required: - terms PhraseQuery: + example: + terms: terms + column: column + slop: 0 properties: column: nullable: true @@ -2835,18 +2989,42 @@ components: required: - terms BoostQuery: + description: Boost query that scores documents matching positive query higher + and negative query lower + example: + negative: null + negative_boost: 7.0614014 + positive: null properties: + positive: + $ref: '#/components/schemas/FtsQuery' negative: - type: object + $ref: '#/components/schemas/FtsQuery' negative_boost: + default: 0.5 + description: "Boost factor for negative query (default: 0.5)" format: float type: number - positive: - type: object required: - negative - positive MultiMatchQuery: + example: + match_queries: + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 properties: match_queries: items: @@ -2856,6 +3034,16 @@ components: - match_queries BooleanQuery: description: "Boolean query with must, should, and must_not clauses" + example: + must_not: + - null + - null + should: + - null + - null + must: + - null + - null properties: must: description: Queries that must match (AND) @@ -2881,34 +3069,63 @@ components: - And - Or type: string - FtsQuery_oneOf: - properties: - match: - $ref: '#/components/schemas/MatchQuery' - required: - - match - FtsQuery_oneOf_1: - properties: - phrase: - $ref: '#/components/schemas/PhraseQuery' - required: - - phrase - FtsQuery_oneOf_2: - properties: - boost: - $ref: '#/components/schemas/BoostQuery' - required: - - boost - FtsQuery_oneOf_3: - properties: - multi_match: - $ref: '#/components/schemas/MultiMatchQuery' - required: - - multi_match - FtsQuery_oneOf_4: - properties: - boolean: - $ref: '#/components/schemas/BooleanQuery' - required: - - boolean + QueryRequest_full_text_query: + description: "Optional full-text search query. Provide either string_query or\ + \ structured_query, not both." + example: + string_query: + columns: + - columns + - columns + query: query + structured_query: + query: + boolean: + must_not: + - null + - null + should: + - null + - null + must: + - null + - null + phrase: + terms: terms + column: column + slop: 0 + match: + fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + boost: + negative: null + negative_boost: 7.0614014 + positive: null + multi_match: + match_queries: + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + - fuzziness: 0 + terms: terms + column: column + boost: 6.0274563 + prefix_length: 0 + operator: And + max_expansions: 0 + properties: + string_query: + $ref: '#/components/schemas/StringFtsQuery' + structured_query: + $ref: '#/components/schemas/StructuredFtsQuery' + nullable: true diff --git a/java/lance-namespace-apache-client/docs/BoostQuery.md b/java/lance-namespace-apache-client/docs/BoostQuery.md index ee8870a61..84a0f560b 100644 --- a/java/lance-namespace-apache-client/docs/BoostQuery.md +++ b/java/lance-namespace-apache-client/docs/BoostQuery.md @@ -2,14 +2,15 @@ # BoostQuery +Boost query that scores documents matching positive query higher and negative query lower ## Properties | Name | Type | Description | Notes | |------------ | ------------- | ------------- | -------------| -|**negative** | **Object** | | | -|**negativeBoost** | **Float** | | [optional] | -|**positive** | **Object** | | | +|**positive** | [**FtsQuery**](FtsQuery.md) | | | +|**negative** | [**FtsQuery**](FtsQuery.md) | | | +|**negativeBoost** | **Float** | Boost factor for negative query (default: 0.5) | [optional] | diff --git a/java/lance-namespace-apache-client/docs/FtsQuery.md b/java/lance-namespace-apache-client/docs/FtsQuery.md index 026f79dbb..f080383b1 100644 --- a/java/lance-namespace-apache-client/docs/FtsQuery.md +++ b/java/lance-namespace-apache-client/docs/FtsQuery.md @@ -2,16 +2,17 @@ # FtsQuery +Full-text search query. Exactly one query type field must be provided. This structure follows the same pattern as AlterTransactionAction to minimize differences and compatibility issues across codegen in different languages. ## Properties | Name | Type | Description | Notes | |------------ | ------------- | ------------- | -------------| -|**match** | [**MatchQuery**](MatchQuery.md) | | | -|**phrase** | [**PhraseQuery**](PhraseQuery.md) | | | -|**boost** | [**BoostQuery**](BoostQuery.md) | | | -|**multiMatch** | [**MultiMatchQuery**](MultiMatchQuery.md) | | | -|**_boolean** | [**BooleanQuery**](BooleanQuery.md) | | | +|**match** | [**MatchQuery**](MatchQuery.md) | | [optional] | +|**phrase** | [**PhraseQuery**](PhraseQuery.md) | | [optional] | +|**boost** | [**BoostQuery**](BoostQuery.md) | | [optional] | +|**multiMatch** | [**MultiMatchQuery**](MultiMatchQuery.md) | | [optional] | +|**_boolean** | [**BooleanQuery**](BooleanQuery.md) | | [optional] | diff --git a/java/lance-namespace-apache-client/docs/FtsQueryOneOf.md b/java/lance-namespace-apache-client/docs/FtsQueryOneOf.md deleted file mode 100644 index edaf162d1..000000000 --- a/java/lance-namespace-apache-client/docs/FtsQueryOneOf.md +++ /dev/null @@ -1,13 +0,0 @@ - - -# FtsQueryOneOf - - -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -|**match** | [**MatchQuery**](MatchQuery.md) | | | - - - diff --git a/java/lance-namespace-apache-client/docs/FtsQueryOneOf1.md b/java/lance-namespace-apache-client/docs/FtsQueryOneOf1.md deleted file mode 100644 index 350449ea5..000000000 --- a/java/lance-namespace-apache-client/docs/FtsQueryOneOf1.md +++ /dev/null @@ -1,13 +0,0 @@ - - -# FtsQueryOneOf1 - - -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -|**phrase** | [**PhraseQuery**](PhraseQuery.md) | | | - - - diff --git a/java/lance-namespace-apache-client/docs/FtsQueryOneOf2.md b/java/lance-namespace-apache-client/docs/FtsQueryOneOf2.md deleted file mode 100644 index 3ee365dc7..000000000 --- a/java/lance-namespace-apache-client/docs/FtsQueryOneOf2.md +++ /dev/null @@ -1,13 +0,0 @@ - - -# FtsQueryOneOf2 - - -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -|**boost** | [**BoostQuery**](BoostQuery.md) | | | - - - diff --git a/java/lance-namespace-apache-client/docs/FtsQueryOneOf3.md b/java/lance-namespace-apache-client/docs/FtsQueryOneOf3.md deleted file mode 100644 index a1cd288b1..000000000 --- a/java/lance-namespace-apache-client/docs/FtsQueryOneOf3.md +++ /dev/null @@ -1,13 +0,0 @@ - - -# FtsQueryOneOf3 - - -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -|**multiMatch** | [**MultiMatchQuery**](MultiMatchQuery.md) | | | - - - diff --git a/java/lance-namespace-apache-client/docs/FtsQueryOneOf4.md b/java/lance-namespace-apache-client/docs/FtsQueryOneOf4.md deleted file mode 100644 index a6674470e..000000000 --- a/java/lance-namespace-apache-client/docs/FtsQueryOneOf4.md +++ /dev/null @@ -1,13 +0,0 @@ - - -# FtsQueryOneOf4 - - -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -|**_boolean** | [**BooleanQuery**](BooleanQuery.md) | | | - - - diff --git a/java/lance-namespace-apache-client/docs/QueryRequest.md b/java/lance-namespace-apache-client/docs/QueryRequest.md index b96947565..d98905487 100644 --- a/java/lance-namespace-apache-client/docs/QueryRequest.md +++ b/java/lance-namespace-apache-client/docs/QueryRequest.md @@ -15,7 +15,7 @@ |**ef** | **Integer** | Search effort parameter for HNSW index | [optional] | |**fastSearch** | **Boolean** | Whether to use fast search | [optional] | |**filter** | **String** | Optional SQL filter expression | [optional] | -|**fullTextQuery** | [**StringFtsQuery**](StringFtsQuery.md) | Optional full-text search query (only string query supported) | [optional] | +|**fullTextQuery** | [**QueryRequestFullTextQuery**](QueryRequestFullTextQuery.md) | | [optional] | |**k** | **Integer** | Number of results to return | | |**lowerBound** | **Float** | Lower bound for search | [optional] | |**nprobes** | **Integer** | Number of probes for IVF index | [optional] | diff --git a/java/lance-namespace-apache-client/docs/QueryRequestFullTextQuery.md b/java/lance-namespace-apache-client/docs/QueryRequestFullTextQuery.md new file mode 100644 index 000000000..cebf0f1ca --- /dev/null +++ b/java/lance-namespace-apache-client/docs/QueryRequestFullTextQuery.md @@ -0,0 +1,15 @@ + + +# QueryRequestFullTextQuery + +Optional full-text search query. Provide either string_query or structured_query, not both. + +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +|**stringQuery** | [**StringFtsQuery**](StringFtsQuery.md) | | [optional] | +|**structuredQuery** | [**StructuredFtsQuery**](StructuredFtsQuery.md) | | [optional] | + + + diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/BoostQuery.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/BoostQuery.java index 32728e6ab..895191147 100644 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/BoostQuery.java +++ b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/BoostQuery.java @@ -22,28 +22,52 @@ import java.util.Objects; import java.util.StringJoiner; -/** BoostQuery */ +/** Boost query that scores documents matching positive query higher and negative query lower */ @JsonPropertyOrder({ + BoostQuery.JSON_PROPERTY_POSITIVE, BoostQuery.JSON_PROPERTY_NEGATIVE, - BoostQuery.JSON_PROPERTY_NEGATIVE_BOOST, - BoostQuery.JSON_PROPERTY_POSITIVE + BoostQuery.JSON_PROPERTY_NEGATIVE_BOOST }) @javax.annotation.Generated( value = "org.openapitools.codegen.languages.JavaClientCodegen", comments = "Generator version: 7.12.0") public class BoostQuery { + public static final String JSON_PROPERTY_POSITIVE = "positive"; + @javax.annotation.Nonnull private FtsQuery positive; + public static final String JSON_PROPERTY_NEGATIVE = "negative"; - @javax.annotation.Nonnull private Object negative; + @javax.annotation.Nonnull private FtsQuery negative; public static final String JSON_PROPERTY_NEGATIVE_BOOST = "negative_boost"; - @javax.annotation.Nullable private Float negativeBoost; - - public static final String JSON_PROPERTY_POSITIVE = "positive"; - @javax.annotation.Nonnull private Object positive; + @javax.annotation.Nullable private Float negativeBoost = 0.5f; public BoostQuery() {} - public BoostQuery negative(@javax.annotation.Nonnull Object negative) { + public BoostQuery positive(@javax.annotation.Nonnull FtsQuery positive) { + + this.positive = positive; + return this; + } + + /** + * Get positive + * + * @return positive + */ + @javax.annotation.Nonnull + @JsonProperty(JSON_PROPERTY_POSITIVE) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public FtsQuery getPositive() { + return positive; + } + + @JsonProperty(JSON_PROPERTY_POSITIVE) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public void setPositive(@javax.annotation.Nonnull FtsQuery positive) { + this.positive = positive; + } + + public BoostQuery negative(@javax.annotation.Nonnull FtsQuery negative) { this.negative = negative; return this; @@ -57,13 +81,13 @@ public BoostQuery negative(@javax.annotation.Nonnull Object negative) { @javax.annotation.Nonnull @JsonProperty(JSON_PROPERTY_NEGATIVE) @JsonInclude(value = JsonInclude.Include.ALWAYS) - public Object getNegative() { + public FtsQuery getNegative() { return negative; } @JsonProperty(JSON_PROPERTY_NEGATIVE) @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setNegative(@javax.annotation.Nonnull Object negative) { + public void setNegative(@javax.annotation.Nonnull FtsQuery negative) { this.negative = negative; } @@ -74,7 +98,7 @@ public BoostQuery negativeBoost(@javax.annotation.Nullable Float negativeBoost) } /** - * Get negativeBoost + * Boost factor for negative query (default: 0.5) * * @return negativeBoost */ @@ -91,30 +115,6 @@ public void setNegativeBoost(@javax.annotation.Nullable Float negativeBoost) { this.negativeBoost = negativeBoost; } - public BoostQuery positive(@javax.annotation.Nonnull Object positive) { - - this.positive = positive; - return this; - } - - /** - * Get positive - * - * @return positive - */ - @javax.annotation.Nonnull - @JsonProperty(JSON_PROPERTY_POSITIVE) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public Object getPositive() { - return positive; - } - - @JsonProperty(JSON_PROPERTY_POSITIVE) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setPositive(@javax.annotation.Nonnull Object positive) { - this.positive = positive; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -124,23 +124,23 @@ public boolean equals(Object o) { return false; } BoostQuery boostQuery = (BoostQuery) o; - return Objects.equals(this.negative, boostQuery.negative) - && Objects.equals(this.negativeBoost, boostQuery.negativeBoost) - && Objects.equals(this.positive, boostQuery.positive); + return Objects.equals(this.positive, boostQuery.positive) + && Objects.equals(this.negative, boostQuery.negative) + && Objects.equals(this.negativeBoost, boostQuery.negativeBoost); } @Override public int hashCode() { - return Objects.hash(negative, negativeBoost, positive); + return Objects.hash(positive, negative, negativeBoost); } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("class BoostQuery {\n"); + sb.append(" positive: ").append(toIndentedString(positive)).append("\n"); sb.append(" negative: ").append(toIndentedString(negative)).append("\n"); sb.append(" negativeBoost: ").append(toIndentedString(negativeBoost)).append("\n"); - sb.append(" positive: ").append(toIndentedString(positive)).append("\n"); sb.append("}"); return sb.toString(); } @@ -187,20 +187,14 @@ public String toUrlQueryString(String prefix) { StringJoiner joiner = new StringJoiner("&"); + // add `positive` to the URL query string + if (getPositive() != null) { + joiner.add(getPositive().toUrlQueryString(prefix + "positive" + suffix)); + } + // add `negative` to the URL query string if (getNegative() != null) { - try { - joiner.add( - String.format( - "%snegative%s=%s", - prefix, - suffix, - URLEncoder.encode(String.valueOf(getNegative()), "UTF-8") - .replaceAll("\\+", "%20"))); - } catch (UnsupportedEncodingException e) { - // Should never happen, UTF-8 is always supported - throw new RuntimeException(e); - } + joiner.add(getNegative().toUrlQueryString(prefix + "negative" + suffix)); } // add `negative_boost` to the URL query string @@ -219,22 +213,6 @@ public String toUrlQueryString(String prefix) { } } - // add `positive` to the URL query string - if (getPositive() != null) { - try { - joiner.add( - String.format( - "%spositive%s=%s", - prefix, - suffix, - URLEncoder.encode(String.valueOf(getPositive()), "UTF-8") - .replaceAll("\\+", "%20"))); - } catch (UnsupportedEncodingException e) { - // Should never happen, UTF-8 is always supported - throw new RuntimeException(e); - } - } - return joiner.toString(); } } diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQuery.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQuery.java index d00f386ea..8a0fb468e 100644 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQuery.java +++ b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQuery.java @@ -20,7 +20,11 @@ import java.util.Objects; import java.util.StringJoiner; -/** FtsQuery */ +/** + * Full-text search query. Exactly one query type field must be provided. This structure follows the + * same pattern as AlterTransactionAction to minimize differences and compatibility issues across + * codegen in different languages. + */ @JsonPropertyOrder({ FtsQuery.JSON_PROPERTY_MATCH, FtsQuery.JSON_PROPERTY_PHRASE, @@ -33,23 +37,23 @@ comments = "Generator version: 7.12.0") public class FtsQuery { public static final String JSON_PROPERTY_MATCH = "match"; - @javax.annotation.Nonnull private MatchQuery match; + @javax.annotation.Nullable private MatchQuery match; public static final String JSON_PROPERTY_PHRASE = "phrase"; - @javax.annotation.Nonnull private PhraseQuery phrase; + @javax.annotation.Nullable private PhraseQuery phrase; public static final String JSON_PROPERTY_BOOST = "boost"; - @javax.annotation.Nonnull private BoostQuery boost; + @javax.annotation.Nullable private BoostQuery boost; public static final String JSON_PROPERTY_MULTI_MATCH = "multi_match"; - @javax.annotation.Nonnull private MultiMatchQuery multiMatch; + @javax.annotation.Nullable private MultiMatchQuery multiMatch; public static final String JSON_PROPERTY_BOOLEAN = "boolean"; - @javax.annotation.Nonnull private BooleanQuery _boolean; + @javax.annotation.Nullable private BooleanQuery _boolean; public FtsQuery() {} - public FtsQuery match(@javax.annotation.Nonnull MatchQuery match) { + public FtsQuery match(@javax.annotation.Nullable MatchQuery match) { this.match = match; return this; @@ -60,20 +64,20 @@ public FtsQuery match(@javax.annotation.Nonnull MatchQuery match) { * * @return match */ - @javax.annotation.Nonnull + @javax.annotation.Nullable @JsonProperty(JSON_PROPERTY_MATCH) - @JsonInclude(value = JsonInclude.Include.ALWAYS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) public MatchQuery getMatch() { return match; } @JsonProperty(JSON_PROPERTY_MATCH) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setMatch(@javax.annotation.Nonnull MatchQuery match) { + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setMatch(@javax.annotation.Nullable MatchQuery match) { this.match = match; } - public FtsQuery phrase(@javax.annotation.Nonnull PhraseQuery phrase) { + public FtsQuery phrase(@javax.annotation.Nullable PhraseQuery phrase) { this.phrase = phrase; return this; @@ -84,20 +88,20 @@ public FtsQuery phrase(@javax.annotation.Nonnull PhraseQuery phrase) { * * @return phrase */ - @javax.annotation.Nonnull + @javax.annotation.Nullable @JsonProperty(JSON_PROPERTY_PHRASE) - @JsonInclude(value = JsonInclude.Include.ALWAYS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) public PhraseQuery getPhrase() { return phrase; } @JsonProperty(JSON_PROPERTY_PHRASE) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setPhrase(@javax.annotation.Nonnull PhraseQuery phrase) { + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setPhrase(@javax.annotation.Nullable PhraseQuery phrase) { this.phrase = phrase; } - public FtsQuery boost(@javax.annotation.Nonnull BoostQuery boost) { + public FtsQuery boost(@javax.annotation.Nullable BoostQuery boost) { this.boost = boost; return this; @@ -108,20 +112,20 @@ public FtsQuery boost(@javax.annotation.Nonnull BoostQuery boost) { * * @return boost */ - @javax.annotation.Nonnull + @javax.annotation.Nullable @JsonProperty(JSON_PROPERTY_BOOST) - @JsonInclude(value = JsonInclude.Include.ALWAYS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) public BoostQuery getBoost() { return boost; } @JsonProperty(JSON_PROPERTY_BOOST) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setBoost(@javax.annotation.Nonnull BoostQuery boost) { + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setBoost(@javax.annotation.Nullable BoostQuery boost) { this.boost = boost; } - public FtsQuery multiMatch(@javax.annotation.Nonnull MultiMatchQuery multiMatch) { + public FtsQuery multiMatch(@javax.annotation.Nullable MultiMatchQuery multiMatch) { this.multiMatch = multiMatch; return this; @@ -132,20 +136,20 @@ public FtsQuery multiMatch(@javax.annotation.Nonnull MultiMatchQuery multiMatch) * * @return multiMatch */ - @javax.annotation.Nonnull + @javax.annotation.Nullable @JsonProperty(JSON_PROPERTY_MULTI_MATCH) - @JsonInclude(value = JsonInclude.Include.ALWAYS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) public MultiMatchQuery getMultiMatch() { return multiMatch; } @JsonProperty(JSON_PROPERTY_MULTI_MATCH) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setMultiMatch(@javax.annotation.Nonnull MultiMatchQuery multiMatch) { + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setMultiMatch(@javax.annotation.Nullable MultiMatchQuery multiMatch) { this.multiMatch = multiMatch; } - public FtsQuery _boolean(@javax.annotation.Nonnull BooleanQuery _boolean) { + public FtsQuery _boolean(@javax.annotation.Nullable BooleanQuery _boolean) { this._boolean = _boolean; return this; @@ -156,16 +160,16 @@ public FtsQuery _boolean(@javax.annotation.Nonnull BooleanQuery _boolean) { * * @return _boolean */ - @javax.annotation.Nonnull + @javax.annotation.Nullable @JsonProperty(JSON_PROPERTY_BOOLEAN) - @JsonInclude(value = JsonInclude.Include.ALWAYS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) public BooleanQuery getBoolean() { return _boolean; } @JsonProperty(JSON_PROPERTY_BOOLEAN) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setBoolean(@javax.annotation.Nonnull BooleanQuery _boolean) { + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setBoolean(@javax.annotation.Nullable BooleanQuery _boolean) { this._boolean = _boolean; } diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf.java deleted file mode 100644 index cb53a2079..000000000 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.annotation.JsonTypeName; - -import java.util.Objects; -import java.util.StringJoiner; - -/** FtsQueryOneOf */ -@JsonPropertyOrder({FtsQueryOneOf.JSON_PROPERTY_MATCH}) -@JsonTypeName("FtsQuery_oneOf") -@javax.annotation.Generated( - value = "org.openapitools.codegen.languages.JavaClientCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf { - public static final String JSON_PROPERTY_MATCH = "match"; - @javax.annotation.Nonnull private MatchQuery match; - - public FtsQueryOneOf() {} - - public FtsQueryOneOf match(@javax.annotation.Nonnull MatchQuery match) { - - this.match = match; - return this; - } - - /** - * Get match - * - * @return match - */ - @javax.annotation.Nonnull - @JsonProperty(JSON_PROPERTY_MATCH) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public MatchQuery getMatch() { - return match; - } - - @JsonProperty(JSON_PROPERTY_MATCH) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setMatch(@javax.annotation.Nonnull MatchQuery match) { - this.match = match; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf ftsQueryOneOf = (FtsQueryOneOf) o; - return Objects.equals(this.match, ftsQueryOneOf.match); - } - - @Override - public int hashCode() { - return Objects.hash(match); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf {\n"); - sb.append(" match: ").append(toIndentedString(match)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } - - /** - * Convert the instance into URL query string. - * - * @return URL query string - */ - public String toUrlQueryString() { - return toUrlQueryString(null); - } - - /** - * Convert the instance into URL query string. - * - * @param prefix prefix of the query string - * @return URL query string - */ - public String toUrlQueryString(String prefix) { - String suffix = ""; - String containerSuffix = ""; - String containerPrefix = ""; - if (prefix == null) { - // style=form, explode=true, e.g. /pet?name=cat&type=manx - prefix = ""; - } else { - // deepObject style e.g. /pet?id[name]=cat&id[type]=manx - prefix = prefix + "["; - suffix = "]"; - containerSuffix = "]"; - containerPrefix = "["; - } - - StringJoiner joiner = new StringJoiner("&"); - - // add `match` to the URL query string - if (getMatch() != null) { - joiner.add(getMatch().toUrlQueryString(prefix + "match" + suffix)); - } - - return joiner.toString(); - } -} diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf1.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf1.java deleted file mode 100644 index 26d20e586..000000000 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf1.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.annotation.JsonTypeName; - -import java.util.Objects; -import java.util.StringJoiner; - -/** FtsQueryOneOf1 */ -@JsonPropertyOrder({FtsQueryOneOf1.JSON_PROPERTY_PHRASE}) -@JsonTypeName("FtsQuery_oneOf_1") -@javax.annotation.Generated( - value = "org.openapitools.codegen.languages.JavaClientCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf1 { - public static final String JSON_PROPERTY_PHRASE = "phrase"; - @javax.annotation.Nonnull private PhraseQuery phrase; - - public FtsQueryOneOf1() {} - - public FtsQueryOneOf1 phrase(@javax.annotation.Nonnull PhraseQuery phrase) { - - this.phrase = phrase; - return this; - } - - /** - * Get phrase - * - * @return phrase - */ - @javax.annotation.Nonnull - @JsonProperty(JSON_PROPERTY_PHRASE) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public PhraseQuery getPhrase() { - return phrase; - } - - @JsonProperty(JSON_PROPERTY_PHRASE) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setPhrase(@javax.annotation.Nonnull PhraseQuery phrase) { - this.phrase = phrase; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf1 ftsQueryOneOf1 = (FtsQueryOneOf1) o; - return Objects.equals(this.phrase, ftsQueryOneOf1.phrase); - } - - @Override - public int hashCode() { - return Objects.hash(phrase); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf1 {\n"); - sb.append(" phrase: ").append(toIndentedString(phrase)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } - - /** - * Convert the instance into URL query string. - * - * @return URL query string - */ - public String toUrlQueryString() { - return toUrlQueryString(null); - } - - /** - * Convert the instance into URL query string. - * - * @param prefix prefix of the query string - * @return URL query string - */ - public String toUrlQueryString(String prefix) { - String suffix = ""; - String containerSuffix = ""; - String containerPrefix = ""; - if (prefix == null) { - // style=form, explode=true, e.g. /pet?name=cat&type=manx - prefix = ""; - } else { - // deepObject style e.g. /pet?id[name]=cat&id[type]=manx - prefix = prefix + "["; - suffix = "]"; - containerSuffix = "]"; - containerPrefix = "["; - } - - StringJoiner joiner = new StringJoiner("&"); - - // add `phrase` to the URL query string - if (getPhrase() != null) { - joiner.add(getPhrase().toUrlQueryString(prefix + "phrase" + suffix)); - } - - return joiner.toString(); - } -} diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf2.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf2.java deleted file mode 100644 index 55ce39e05..000000000 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf2.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.annotation.JsonTypeName; - -import java.util.Objects; -import java.util.StringJoiner; - -/** FtsQueryOneOf2 */ -@JsonPropertyOrder({FtsQueryOneOf2.JSON_PROPERTY_BOOST}) -@JsonTypeName("FtsQuery_oneOf_2") -@javax.annotation.Generated( - value = "org.openapitools.codegen.languages.JavaClientCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf2 { - public static final String JSON_PROPERTY_BOOST = "boost"; - @javax.annotation.Nonnull private BoostQuery boost; - - public FtsQueryOneOf2() {} - - public FtsQueryOneOf2 boost(@javax.annotation.Nonnull BoostQuery boost) { - - this.boost = boost; - return this; - } - - /** - * Get boost - * - * @return boost - */ - @javax.annotation.Nonnull - @JsonProperty(JSON_PROPERTY_BOOST) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public BoostQuery getBoost() { - return boost; - } - - @JsonProperty(JSON_PROPERTY_BOOST) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setBoost(@javax.annotation.Nonnull BoostQuery boost) { - this.boost = boost; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf2 ftsQueryOneOf2 = (FtsQueryOneOf2) o; - return Objects.equals(this.boost, ftsQueryOneOf2.boost); - } - - @Override - public int hashCode() { - return Objects.hash(boost); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf2 {\n"); - sb.append(" boost: ").append(toIndentedString(boost)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } - - /** - * Convert the instance into URL query string. - * - * @return URL query string - */ - public String toUrlQueryString() { - return toUrlQueryString(null); - } - - /** - * Convert the instance into URL query string. - * - * @param prefix prefix of the query string - * @return URL query string - */ - public String toUrlQueryString(String prefix) { - String suffix = ""; - String containerSuffix = ""; - String containerPrefix = ""; - if (prefix == null) { - // style=form, explode=true, e.g. /pet?name=cat&type=manx - prefix = ""; - } else { - // deepObject style e.g. /pet?id[name]=cat&id[type]=manx - prefix = prefix + "["; - suffix = "]"; - containerSuffix = "]"; - containerPrefix = "["; - } - - StringJoiner joiner = new StringJoiner("&"); - - // add `boost` to the URL query string - if (getBoost() != null) { - joiner.add(getBoost().toUrlQueryString(prefix + "boost" + suffix)); - } - - return joiner.toString(); - } -} diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf3.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf3.java deleted file mode 100644 index aeed9b66b..000000000 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf3.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.annotation.JsonTypeName; - -import java.util.Objects; -import java.util.StringJoiner; - -/** FtsQueryOneOf3 */ -@JsonPropertyOrder({FtsQueryOneOf3.JSON_PROPERTY_MULTI_MATCH}) -@JsonTypeName("FtsQuery_oneOf_3") -@javax.annotation.Generated( - value = "org.openapitools.codegen.languages.JavaClientCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf3 { - public static final String JSON_PROPERTY_MULTI_MATCH = "multi_match"; - @javax.annotation.Nonnull private MultiMatchQuery multiMatch; - - public FtsQueryOneOf3() {} - - public FtsQueryOneOf3 multiMatch(@javax.annotation.Nonnull MultiMatchQuery multiMatch) { - - this.multiMatch = multiMatch; - return this; - } - - /** - * Get multiMatch - * - * @return multiMatch - */ - @javax.annotation.Nonnull - @JsonProperty(JSON_PROPERTY_MULTI_MATCH) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public MultiMatchQuery getMultiMatch() { - return multiMatch; - } - - @JsonProperty(JSON_PROPERTY_MULTI_MATCH) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setMultiMatch(@javax.annotation.Nonnull MultiMatchQuery multiMatch) { - this.multiMatch = multiMatch; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf3 ftsQueryOneOf3 = (FtsQueryOneOf3) o; - return Objects.equals(this.multiMatch, ftsQueryOneOf3.multiMatch); - } - - @Override - public int hashCode() { - return Objects.hash(multiMatch); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf3 {\n"); - sb.append(" multiMatch: ").append(toIndentedString(multiMatch)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } - - /** - * Convert the instance into URL query string. - * - * @return URL query string - */ - public String toUrlQueryString() { - return toUrlQueryString(null); - } - - /** - * Convert the instance into URL query string. - * - * @param prefix prefix of the query string - * @return URL query string - */ - public String toUrlQueryString(String prefix) { - String suffix = ""; - String containerSuffix = ""; - String containerPrefix = ""; - if (prefix == null) { - // style=form, explode=true, e.g. /pet?name=cat&type=manx - prefix = ""; - } else { - // deepObject style e.g. /pet?id[name]=cat&id[type]=manx - prefix = prefix + "["; - suffix = "]"; - containerSuffix = "]"; - containerPrefix = "["; - } - - StringJoiner joiner = new StringJoiner("&"); - - // add `multi_match` to the URL query string - if (getMultiMatch() != null) { - joiner.add(getMultiMatch().toUrlQueryString(prefix + "multi_match" + suffix)); - } - - return joiner.toString(); - } -} diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf4.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf4.java deleted file mode 100644 index f89133dea..000000000 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/FtsQueryOneOf4.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.model; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.annotation.JsonTypeName; - -import java.util.Objects; -import java.util.StringJoiner; - -/** FtsQueryOneOf4 */ -@JsonPropertyOrder({FtsQueryOneOf4.JSON_PROPERTY_BOOLEAN}) -@JsonTypeName("FtsQuery_oneOf_4") -@javax.annotation.Generated( - value = "org.openapitools.codegen.languages.JavaClientCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf4 { - public static final String JSON_PROPERTY_BOOLEAN = "boolean"; - @javax.annotation.Nonnull private BooleanQuery _boolean; - - public FtsQueryOneOf4() {} - - public FtsQueryOneOf4 _boolean(@javax.annotation.Nonnull BooleanQuery _boolean) { - - this._boolean = _boolean; - return this; - } - - /** - * Get _boolean - * - * @return _boolean - */ - @javax.annotation.Nonnull - @JsonProperty(JSON_PROPERTY_BOOLEAN) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public BooleanQuery getBoolean() { - return _boolean; - } - - @JsonProperty(JSON_PROPERTY_BOOLEAN) - @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setBoolean(@javax.annotation.Nonnull BooleanQuery _boolean) { - this._boolean = _boolean; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf4 ftsQueryOneOf4 = (FtsQueryOneOf4) o; - return Objects.equals(this._boolean, ftsQueryOneOf4._boolean); - } - - @Override - public int hashCode() { - return Objects.hash(_boolean); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf4 {\n"); - sb.append(" _boolean: ").append(toIndentedString(_boolean)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } - - /** - * Convert the instance into URL query string. - * - * @return URL query string - */ - public String toUrlQueryString() { - return toUrlQueryString(null); - } - - /** - * Convert the instance into URL query string. - * - * @param prefix prefix of the query string - * @return URL query string - */ - public String toUrlQueryString(String prefix) { - String suffix = ""; - String containerSuffix = ""; - String containerPrefix = ""; - if (prefix == null) { - // style=form, explode=true, e.g. /pet?name=cat&type=manx - prefix = ""; - } else { - // deepObject style e.g. /pet?id[name]=cat&id[type]=manx - prefix = prefix + "["; - suffix = "]"; - containerSuffix = "]"; - containerPrefix = "["; - } - - StringJoiner joiner = new StringJoiner("&"); - - // add `boolean` to the URL query string - if (getBoolean() != null) { - joiner.add(getBoolean().toUrlQueryString(prefix + "boolean" + suffix)); - } - - return joiner.toString(); - } -} diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequest.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequest.java index 16c5457e7..3937b045d 100644 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequest.java +++ b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequest.java @@ -85,7 +85,7 @@ public class QueryRequest { @javax.annotation.Nullable private JsonNullable filter = JsonNullable.undefined(); public static final String JSON_PROPERTY_FULL_TEXT_QUERY = "full_text_query"; - @javax.annotation.Nullable private StringFtsQuery fullTextQuery; + @javax.annotation.Nullable private QueryRequestFullTextQuery fullTextQuery; public static final String JSON_PROPERTY_K = "k"; @javax.annotation.Nonnull private Integer k; @@ -386,27 +386,28 @@ public void setFilter(@javax.annotation.Nullable String filter) { this.filter = JsonNullable.of(filter); } - public QueryRequest fullTextQuery(@javax.annotation.Nullable StringFtsQuery fullTextQuery) { + public QueryRequest fullTextQuery( + @javax.annotation.Nullable QueryRequestFullTextQuery fullTextQuery) { this.fullTextQuery = fullTextQuery; return this; } /** - * Optional full-text search query (only string query supported) + * Get fullTextQuery * * @return fullTextQuery */ @javax.annotation.Nullable @JsonProperty(JSON_PROPERTY_FULL_TEXT_QUERY) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public StringFtsQuery getFullTextQuery() { + public QueryRequestFullTextQuery getFullTextQuery() { return fullTextQuery; } @JsonProperty(JSON_PROPERTY_FULL_TEXT_QUERY) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public void setFullTextQuery(@javax.annotation.Nullable StringFtsQuery fullTextQuery) { + public void setFullTextQuery(@javax.annotation.Nullable QueryRequestFullTextQuery fullTextQuery) { this.fullTextQuery = fullTextQuery; } diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequestFullTextQuery.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequestFullTextQuery.java new file mode 100644 index 000000000..99d18d0be --- /dev/null +++ b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequestFullTextQuery.java @@ -0,0 +1,174 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonTypeName; + +import java.util.Objects; +import java.util.StringJoiner; + +/** Optional full-text search query. Provide either string_query or structured_query, not both. */ +@JsonPropertyOrder({ + QueryRequestFullTextQuery.JSON_PROPERTY_STRING_QUERY, + QueryRequestFullTextQuery.JSON_PROPERTY_STRUCTURED_QUERY +}) +@JsonTypeName("QueryRequest_full_text_query") +@javax.annotation.Generated( + value = "org.openapitools.codegen.languages.JavaClientCodegen", + comments = "Generator version: 7.12.0") +public class QueryRequestFullTextQuery { + public static final String JSON_PROPERTY_STRING_QUERY = "string_query"; + @javax.annotation.Nullable private StringFtsQuery stringQuery; + + public static final String JSON_PROPERTY_STRUCTURED_QUERY = "structured_query"; + @javax.annotation.Nullable private StructuredFtsQuery structuredQuery; + + public QueryRequestFullTextQuery() {} + + public QueryRequestFullTextQuery stringQuery( + @javax.annotation.Nullable StringFtsQuery stringQuery) { + + this.stringQuery = stringQuery; + return this; + } + + /** + * Get stringQuery + * + * @return stringQuery + */ + @javax.annotation.Nullable + @JsonProperty(JSON_PROPERTY_STRING_QUERY) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public StringFtsQuery getStringQuery() { + return stringQuery; + } + + @JsonProperty(JSON_PROPERTY_STRING_QUERY) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setStringQuery(@javax.annotation.Nullable StringFtsQuery stringQuery) { + this.stringQuery = stringQuery; + } + + public QueryRequestFullTextQuery structuredQuery( + @javax.annotation.Nullable StructuredFtsQuery structuredQuery) { + + this.structuredQuery = structuredQuery; + return this; + } + + /** + * Get structuredQuery + * + * @return structuredQuery + */ + @javax.annotation.Nullable + @JsonProperty(JSON_PROPERTY_STRUCTURED_QUERY) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public StructuredFtsQuery getStructuredQuery() { + return structuredQuery; + } + + @JsonProperty(JSON_PROPERTY_STRUCTURED_QUERY) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setStructuredQuery(@javax.annotation.Nullable StructuredFtsQuery structuredQuery) { + this.structuredQuery = structuredQuery; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + QueryRequestFullTextQuery queryRequestFullTextQuery = (QueryRequestFullTextQuery) o; + return Objects.equals(this.stringQuery, queryRequestFullTextQuery.stringQuery) + && Objects.equals(this.structuredQuery, queryRequestFullTextQuery.structuredQuery); + } + + @Override + public int hashCode() { + return Objects.hash(stringQuery, structuredQuery); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class QueryRequestFullTextQuery {\n"); + sb.append(" stringQuery: ").append(toIndentedString(stringQuery)).append("\n"); + sb.append(" structuredQuery: ").append(toIndentedString(structuredQuery)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + + /** + * Convert the instance into URL query string. + * + * @return URL query string + */ + public String toUrlQueryString() { + return toUrlQueryString(null); + } + + /** + * Convert the instance into URL query string. + * + * @param prefix prefix of the query string + * @return URL query string + */ + public String toUrlQueryString(String prefix) { + String suffix = ""; + String containerSuffix = ""; + String containerPrefix = ""; + if (prefix == null) { + // style=form, explode=true, e.g. /pet?name=cat&type=manx + prefix = ""; + } else { + // deepObject style e.g. /pet?id[name]=cat&id[type]=manx + prefix = prefix + "["; + suffix = "]"; + containerSuffix = "]"; + containerPrefix = "["; + } + + StringJoiner joiner = new StringJoiner("&"); + + // add `string_query` to the URL query string + if (getStringQuery() != null) { + joiner.add(getStringQuery().toUrlQueryString(prefix + "string_query" + suffix)); + } + + // add `structured_query` to the URL query string + if (getStructuredQuery() != null) { + joiner.add(getStructuredQuery().toUrlQueryString(prefix + "structured_query" + suffix)); + } + + return joiner.toString(); + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/RestTableApiTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/RestTableApiTest.java index b10bd6c80..338539c06 100644 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/RestTableApiTest.java +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/RestTableApiTest.java @@ -654,7 +654,9 @@ public void testFtsQuery() throws IOException { StringFtsQuery simpleFts = new StringFtsQuery(); simpleFts.setQuery("document"); - stringFtsQuery.setFullTextQuery(simpleFts); + QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); + fullTextQuery.setStringQuery(simpleFts); + stringFtsQuery.setFullTextQuery(fullTextQuery); byte[] ftsResult = namespace.queryTable(stringFtsQuery); assertNotNull(ftsResult, "FTS query result should not be null"); @@ -666,7 +668,7 @@ public void testFtsQuery() throws IOException { ftsPrefilterQuery.setK(5); ftsPrefilterQuery.setPrefilter(true); ftsPrefilterQuery.setFilter("id < 25"); - ftsPrefilterQuery.setFullTextQuery(simpleFts); + ftsPrefilterQuery.setFullTextQuery(fullTextQuery); byte[] ftsPrefilterResult = namespace.queryTable(ftsPrefilterQuery); assertNotNull(ftsPrefilterResult, "FTS prefilter query result should not be null"); @@ -870,7 +872,9 @@ public void testHybridSearch() throws IOException, InterruptedException { // Add full-text search StringFtsQuery ftsQuery = new StringFtsQuery(); ftsQuery.setQuery("document"); - hybridQuery.setFullTextQuery(ftsQuery); + QueryRequestFullTextQuery hybridFullTextQuery = new QueryRequestFullTextQuery(); + hybridFullTextQuery.setStringQuery(ftsQuery); + hybridQuery.setFullTextQuery(hybridFullTextQuery); byte[] hybridResult = namespace.queryTable(hybridQuery); assertNotNull(hybridResult, "Hybrid search result should not be null"); @@ -892,7 +896,9 @@ public void testHybridSearch() throws IOException, InterruptedException { // Add full-text search StringFtsQuery ftsQuery2 = new StringFtsQuery(); ftsQuery2.setQuery("entry"); - hybridFilterQuery.setFullTextQuery(ftsQuery2); + QueryRequestFullTextQuery hybridFilterFullTextQuery = new QueryRequestFullTextQuery(); + hybridFilterFullTextQuery.setStringQuery(ftsQuery2); + hybridFilterQuery.setFullTextQuery(hybridFilterFullTextQuery); // Add filter hybridFilterQuery.setFilter("id < 30"); @@ -918,7 +924,9 @@ public void testHybridSearch() throws IOException, InterruptedException { StringFtsQuery ftsQuery3 = new StringFtsQuery(); ftsQuery3.setQuery("test"); ftsQuery3.setColumns(Arrays.asList("text")); // Search only in text column - hybridColumnsQuery.setFullTextQuery(ftsQuery3); + QueryRequestFullTextQuery hybridColumnsFullTextQuery = new QueryRequestFullTextQuery(); + hybridColumnsFullTextQuery.setStringQuery(ftsQuery3); + hybridColumnsQuery.setFullTextQuery(hybridColumnsFullTextQuery); // Return specific columns hybridColumnsQuery.setColumns(Arrays.asList("id", "text")); diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/BooleanQuery.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/BooleanQuery.java index 4c3ac8c29..c862478a2 100644 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/BooleanQuery.java +++ b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/BooleanQuery.java @@ -34,24 +34,25 @@ comments = "Generator version: 7.12.0") public class BooleanQuery { - @Valid private List must = new ArrayList<>(); + @Valid private List<@Valid FtsQuery> must = new ArrayList<>(); - @Valid private List mustNot = new ArrayList<>(); + @Valid private List<@Valid FtsQuery> mustNot = new ArrayList<>(); - @Valid private List should = new ArrayList<>(); + @Valid private List<@Valid FtsQuery> should = new ArrayList<>(); public BooleanQuery() { super(); } /** Constructor with only required parameters */ - public BooleanQuery(List must, List mustNot, List should) { + public BooleanQuery( + List<@Valid FtsQuery> must, List<@Valid FtsQuery> mustNot, List<@Valid FtsQuery> should) { this.must = must; this.mustNot = mustNot; this.should = should; } - public BooleanQuery must(List must) { + public BooleanQuery must(List<@Valid FtsQuery> must) { this.must = must; return this; } @@ -76,15 +77,15 @@ public BooleanQuery addMustItem(FtsQuery mustItem) { description = "Queries that must match (AND)", requiredMode = Schema.RequiredMode.REQUIRED) @JsonProperty("must") - public List getMust() { + public List<@Valid FtsQuery> getMust() { return must; } - public void setMust(List must) { + public void setMust(List<@Valid FtsQuery> must) { this.must = must; } - public BooleanQuery mustNot(List mustNot) { + public BooleanQuery mustNot(List<@Valid FtsQuery> mustNot) { this.mustNot = mustNot; return this; } @@ -109,15 +110,15 @@ public BooleanQuery addMustNotItem(FtsQuery mustNotItem) { description = "Queries that must not match (NOT)", requiredMode = Schema.RequiredMode.REQUIRED) @JsonProperty("must_not") - public List getMustNot() { + public List<@Valid FtsQuery> getMustNot() { return mustNot; } - public void setMustNot(List mustNot) { + public void setMustNot(List<@Valid FtsQuery> mustNot) { this.mustNot = mustNot; } - public BooleanQuery should(List should) { + public BooleanQuery should(List<@Valid FtsQuery> should) { this.should = should; return this; } @@ -142,11 +143,11 @@ public BooleanQuery addShouldItem(FtsQuery shouldItem) { description = "Queries that should match (OR)", requiredMode = Schema.RequiredMode.REQUIRED) @JsonProperty("should") - public List getShould() { + public List<@Valid FtsQuery> getShould() { return should; } - public void setShould(List should) { + public void setShould(List<@Valid FtsQuery> should) { this.should = should; } diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/BoostQuery.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/BoostQuery.java index bc3e92571..3421e688e 100644 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/BoostQuery.java +++ b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/BoostQuery.java @@ -17,34 +17,61 @@ import io.swagger.v3.oas.annotations.media.Schema; import javax.annotation.Generated; +import javax.validation.Valid; import javax.validation.constraints.*; import java.util.*; import java.util.Objects; -/** BoostQuery */ +/** Boost query that scores documents matching positive query higher and negative query lower */ +@Schema( + name = "BoostQuery", + description = + "Boost query that scores documents matching positive query higher and negative query lower") @Generated( value = "org.openapitools.codegen.languages.SpringCodegen", comments = "Generator version: 7.12.0") public class BoostQuery { - private Object negative; + private FtsQuery positive; - private Float negativeBoost; + private FtsQuery negative; - private Object positive; + private Float negativeBoost = 0.5f; public BoostQuery() { super(); } /** Constructor with only required parameters */ - public BoostQuery(Object negative, Object positive) { + public BoostQuery(FtsQuery positive, FtsQuery negative) { + this.positive = positive; this.negative = negative; + } + + public BoostQuery positive(FtsQuery positive) { this.positive = positive; + return this; } - public BoostQuery negative(Object negative) { + /** + * Get positive + * + * @return positive + */ + @NotNull + @Valid + @Schema(name = "positive", requiredMode = Schema.RequiredMode.REQUIRED) + @JsonProperty("positive") + public FtsQuery getPositive() { + return positive; + } + + public void setPositive(FtsQuery positive) { + this.positive = positive; + } + + public BoostQuery negative(FtsQuery negative) { this.negative = negative; return this; } @@ -55,13 +82,14 @@ public BoostQuery negative(Object negative) { * @return negative */ @NotNull + @Valid @Schema(name = "negative", requiredMode = Schema.RequiredMode.REQUIRED) @JsonProperty("negative") - public Object getNegative() { + public FtsQuery getNegative() { return negative; } - public void setNegative(Object negative) { + public void setNegative(FtsQuery negative) { this.negative = negative; } @@ -71,11 +99,14 @@ public BoostQuery negativeBoost(Float negativeBoost) { } /** - * Get negativeBoost + * Boost factor for negative query (default: 0.5) * * @return negativeBoost */ - @Schema(name = "negative_boost", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @Schema( + name = "negative_boost", + description = "Boost factor for negative query (default: 0.5)", + requiredMode = Schema.RequiredMode.NOT_REQUIRED) @JsonProperty("negative_boost") public Float getNegativeBoost() { return negativeBoost; @@ -85,27 +116,6 @@ public void setNegativeBoost(Float negativeBoost) { this.negativeBoost = negativeBoost; } - public BoostQuery positive(Object positive) { - this.positive = positive; - return this; - } - - /** - * Get positive - * - * @return positive - */ - @NotNull - @Schema(name = "positive", requiredMode = Schema.RequiredMode.REQUIRED) - @JsonProperty("positive") - public Object getPositive() { - return positive; - } - - public void setPositive(Object positive) { - this.positive = positive; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -115,23 +125,23 @@ public boolean equals(Object o) { return false; } BoostQuery boostQuery = (BoostQuery) o; - return Objects.equals(this.negative, boostQuery.negative) - && Objects.equals(this.negativeBoost, boostQuery.negativeBoost) - && Objects.equals(this.positive, boostQuery.positive); + return Objects.equals(this.positive, boostQuery.positive) + && Objects.equals(this.negative, boostQuery.negative) + && Objects.equals(this.negativeBoost, boostQuery.negativeBoost); } @Override public int hashCode() { - return Objects.hash(negative, negativeBoost, positive); + return Objects.hash(positive, negative, negativeBoost); } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("class BoostQuery {\n"); + sb.append(" positive: ").append(toIndentedString(positive)).append("\n"); sb.append(" negative: ").append(toIndentedString(negative)).append("\n"); sb.append(" negativeBoost: ").append(toIndentedString(negativeBoost)).append("\n"); - sb.append(" positive: ").append(toIndentedString(positive)).append("\n"); sb.append("}"); return sb.toString(); } diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQuery.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQuery.java index 87aff45ca..1c55168c1 100644 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQuery.java +++ b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQuery.java @@ -13,12 +13,186 @@ */ package com.lancedb.lance.namespace.server.springboot.model; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + import javax.annotation.Generated; +import javax.validation.Valid; import javax.validation.constraints.*; import java.util.*; +import java.util.Objects; +/** + * Full-text search query. Exactly one query type field must be provided. This structure follows the + * same pattern as AlterTransactionAction to minimize differences and compatibility issues across + * codegen in different languages. + */ +@Schema( + name = "FtsQuery", + description = + "Full-text search query. Exactly one query type field must be provided. This structure follows the same pattern as AlterTransactionAction to minimize differences and compatibility issues across codegen in different languages. ") @Generated( value = "org.openapitools.codegen.languages.SpringCodegen", comments = "Generator version: 7.12.0") -public interface FtsQuery {} +public class FtsQuery { + + private MatchQuery match; + + private PhraseQuery phrase; + + private BoostQuery boost; + + private MultiMatchQuery multiMatch; + + private BooleanQuery _boolean; + + public FtsQuery match(MatchQuery match) { + this.match = match; + return this; + } + + /** + * Get match + * + * @return match + */ + @Valid + @Schema(name = "match", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("match") + public MatchQuery getMatch() { + return match; + } + + public void setMatch(MatchQuery match) { + this.match = match; + } + + public FtsQuery phrase(PhraseQuery phrase) { + this.phrase = phrase; + return this; + } + + /** + * Get phrase + * + * @return phrase + */ + @Valid + @Schema(name = "phrase", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("phrase") + public PhraseQuery getPhrase() { + return phrase; + } + + public void setPhrase(PhraseQuery phrase) { + this.phrase = phrase; + } + + public FtsQuery boost(BoostQuery boost) { + this.boost = boost; + return this; + } + + /** + * Get boost + * + * @return boost + */ + @Valid + @Schema(name = "boost", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("boost") + public BoostQuery getBoost() { + return boost; + } + + public void setBoost(BoostQuery boost) { + this.boost = boost; + } + + public FtsQuery multiMatch(MultiMatchQuery multiMatch) { + this.multiMatch = multiMatch; + return this; + } + + /** + * Get multiMatch + * + * @return multiMatch + */ + @Valid + @Schema(name = "multi_match", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("multi_match") + public MultiMatchQuery getMultiMatch() { + return multiMatch; + } + + public void setMultiMatch(MultiMatchQuery multiMatch) { + this.multiMatch = multiMatch; + } + + public FtsQuery _boolean(BooleanQuery _boolean) { + this._boolean = _boolean; + return this; + } + + /** + * Get _boolean + * + * @return _boolean + */ + @Valid + @Schema(name = "boolean", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("boolean") + public BooleanQuery getBoolean() { + return _boolean; + } + + public void setBoolean(BooleanQuery _boolean) { + this._boolean = _boolean; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FtsQuery ftsQuery = (FtsQuery) o; + return Objects.equals(this.match, ftsQuery.match) + && Objects.equals(this.phrase, ftsQuery.phrase) + && Objects.equals(this.boost, ftsQuery.boost) + && Objects.equals(this.multiMatch, ftsQuery.multiMatch) + && Objects.equals(this._boolean, ftsQuery._boolean); + } + + @Override + public int hashCode() { + return Objects.hash(match, phrase, boost, multiMatch, _boolean); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class FtsQuery {\n"); + sb.append(" match: ").append(toIndentedString(match)).append("\n"); + sb.append(" phrase: ").append(toIndentedString(phrase)).append("\n"); + sb.append(" boost: ").append(toIndentedString(boost)).append("\n"); + sb.append(" multiMatch: ").append(toIndentedString(multiMatch)).append("\n"); + sb.append(" _boolean: ").append(toIndentedString(_boolean)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf.java deleted file mode 100644 index 55d1ad3bc..000000000 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.server.springboot.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import io.swagger.v3.oas.annotations.media.Schema; - -import javax.annotation.Generated; -import javax.validation.Valid; -import javax.validation.constraints.*; - -import java.util.*; -import java.util.Objects; - -/** FtsQueryOneOf */ -@JsonTypeName("FtsQuery_oneOf") -@Generated( - value = "org.openapitools.codegen.languages.SpringCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf implements FtsQuery { - - private MatchQuery match; - - public FtsQueryOneOf() { - super(); - } - - /** Constructor with only required parameters */ - public FtsQueryOneOf(MatchQuery match) { - this.match = match; - } - - public FtsQueryOneOf match(MatchQuery match) { - this.match = match; - return this; - } - - /** - * Get match - * - * @return match - */ - @NotNull - @Valid - @Schema(name = "match", requiredMode = Schema.RequiredMode.REQUIRED) - @JsonProperty("match") - public MatchQuery getMatch() { - return match; - } - - public void setMatch(MatchQuery match) { - this.match = match; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf ftsQueryOneOf = (FtsQueryOneOf) o; - return Objects.equals(this.match, ftsQueryOneOf.match); - } - - @Override - public int hashCode() { - return Objects.hash(match); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf {\n"); - sb.append(" match: ").append(toIndentedString(match)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } -} diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf1.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf1.java deleted file mode 100644 index b8670e8c8..000000000 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf1.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.server.springboot.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import io.swagger.v3.oas.annotations.media.Schema; - -import javax.annotation.Generated; -import javax.validation.Valid; -import javax.validation.constraints.*; - -import java.util.*; -import java.util.Objects; - -/** FtsQueryOneOf1 */ -@JsonTypeName("FtsQuery_oneOf_1") -@Generated( - value = "org.openapitools.codegen.languages.SpringCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf1 implements FtsQuery { - - private PhraseQuery phrase; - - public FtsQueryOneOf1() { - super(); - } - - /** Constructor with only required parameters */ - public FtsQueryOneOf1(PhraseQuery phrase) { - this.phrase = phrase; - } - - public FtsQueryOneOf1 phrase(PhraseQuery phrase) { - this.phrase = phrase; - return this; - } - - /** - * Get phrase - * - * @return phrase - */ - @NotNull - @Valid - @Schema(name = "phrase", requiredMode = Schema.RequiredMode.REQUIRED) - @JsonProperty("phrase") - public PhraseQuery getPhrase() { - return phrase; - } - - public void setPhrase(PhraseQuery phrase) { - this.phrase = phrase; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf1 ftsQueryOneOf1 = (FtsQueryOneOf1) o; - return Objects.equals(this.phrase, ftsQueryOneOf1.phrase); - } - - @Override - public int hashCode() { - return Objects.hash(phrase); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf1 {\n"); - sb.append(" phrase: ").append(toIndentedString(phrase)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } -} diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf2.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf2.java deleted file mode 100644 index fc4dcbb04..000000000 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf2.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.server.springboot.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import io.swagger.v3.oas.annotations.media.Schema; - -import javax.annotation.Generated; -import javax.validation.Valid; -import javax.validation.constraints.*; - -import java.util.*; -import java.util.Objects; - -/** FtsQueryOneOf2 */ -@JsonTypeName("FtsQuery_oneOf_2") -@Generated( - value = "org.openapitools.codegen.languages.SpringCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf2 implements FtsQuery { - - private BoostQuery boost; - - public FtsQueryOneOf2() { - super(); - } - - /** Constructor with only required parameters */ - public FtsQueryOneOf2(BoostQuery boost) { - this.boost = boost; - } - - public FtsQueryOneOf2 boost(BoostQuery boost) { - this.boost = boost; - return this; - } - - /** - * Get boost - * - * @return boost - */ - @NotNull - @Valid - @Schema(name = "boost", requiredMode = Schema.RequiredMode.REQUIRED) - @JsonProperty("boost") - public BoostQuery getBoost() { - return boost; - } - - public void setBoost(BoostQuery boost) { - this.boost = boost; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf2 ftsQueryOneOf2 = (FtsQueryOneOf2) o; - return Objects.equals(this.boost, ftsQueryOneOf2.boost); - } - - @Override - public int hashCode() { - return Objects.hash(boost); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf2 {\n"); - sb.append(" boost: ").append(toIndentedString(boost)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } -} diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf3.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf3.java deleted file mode 100644 index 2955bb986..000000000 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf3.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.server.springboot.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import io.swagger.v3.oas.annotations.media.Schema; - -import javax.annotation.Generated; -import javax.validation.Valid; -import javax.validation.constraints.*; - -import java.util.*; -import java.util.Objects; - -/** FtsQueryOneOf3 */ -@JsonTypeName("FtsQuery_oneOf_3") -@Generated( - value = "org.openapitools.codegen.languages.SpringCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf3 implements FtsQuery { - - private MultiMatchQuery multiMatch; - - public FtsQueryOneOf3() { - super(); - } - - /** Constructor with only required parameters */ - public FtsQueryOneOf3(MultiMatchQuery multiMatch) { - this.multiMatch = multiMatch; - } - - public FtsQueryOneOf3 multiMatch(MultiMatchQuery multiMatch) { - this.multiMatch = multiMatch; - return this; - } - - /** - * Get multiMatch - * - * @return multiMatch - */ - @NotNull - @Valid - @Schema(name = "multi_match", requiredMode = Schema.RequiredMode.REQUIRED) - @JsonProperty("multi_match") - public MultiMatchQuery getMultiMatch() { - return multiMatch; - } - - public void setMultiMatch(MultiMatchQuery multiMatch) { - this.multiMatch = multiMatch; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf3 ftsQueryOneOf3 = (FtsQueryOneOf3) o; - return Objects.equals(this.multiMatch, ftsQueryOneOf3.multiMatch); - } - - @Override - public int hashCode() { - return Objects.hash(multiMatch); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf3 {\n"); - sb.append(" multiMatch: ").append(toIndentedString(multiMatch)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } -} diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf4.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf4.java deleted file mode 100644 index 219bb7452..000000000 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/FtsQueryOneOf4.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace.server.springboot.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import io.swagger.v3.oas.annotations.media.Schema; - -import javax.annotation.Generated; -import javax.validation.Valid; -import javax.validation.constraints.*; - -import java.util.*; -import java.util.Objects; - -/** FtsQueryOneOf4 */ -@JsonTypeName("FtsQuery_oneOf_4") -@Generated( - value = "org.openapitools.codegen.languages.SpringCodegen", - comments = "Generator version: 7.12.0") -public class FtsQueryOneOf4 implements FtsQuery { - - private BooleanQuery _boolean; - - public FtsQueryOneOf4() { - super(); - } - - /** Constructor with only required parameters */ - public FtsQueryOneOf4(BooleanQuery _boolean) { - this._boolean = _boolean; - } - - public FtsQueryOneOf4 _boolean(BooleanQuery _boolean) { - this._boolean = _boolean; - return this; - } - - /** - * Get _boolean - * - * @return _boolean - */ - @NotNull - @Valid - @Schema(name = "boolean", requiredMode = Schema.RequiredMode.REQUIRED) - @JsonProperty("boolean") - public BooleanQuery getBoolean() { - return _boolean; - } - - public void setBoolean(BooleanQuery _boolean) { - this._boolean = _boolean; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FtsQueryOneOf4 ftsQueryOneOf4 = (FtsQueryOneOf4) o; - return Objects.equals(this._boolean, ftsQueryOneOf4._boolean); - } - - @Override - public int hashCode() { - return Objects.hash(_boolean); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class FtsQueryOneOf4 {\n"); - sb.append(" _boolean: ").append(toIndentedString(_boolean)).append("\n"); - sb.append("}"); - return sb.toString(); - } - - /** - * Convert the given object to string with each line indented by 4 spaces (except the first line). - */ - private String toIndentedString(Object o) { - if (o == null) { - return "null"; - } - return o.toString().replace("\n", "\n "); - } -} diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequest.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequest.java index 646695b7f..c033d6854 100644 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequest.java +++ b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequest.java @@ -47,7 +47,7 @@ public class QueryRequest { private String filter = null; - private StringFtsQuery fullTextQuery; + private QueryRequestFullTextQuery fullTextQuery; private Integer k; @@ -280,27 +280,24 @@ public void setFilter(String filter) { this.filter = filter; } - public QueryRequest fullTextQuery(StringFtsQuery fullTextQuery) { + public QueryRequest fullTextQuery(QueryRequestFullTextQuery fullTextQuery) { this.fullTextQuery = fullTextQuery; return this; } /** - * Optional full-text search query (only string query supported) + * Get fullTextQuery * * @return fullTextQuery */ @Valid - @Schema( - name = "full_text_query", - description = "Optional full-text search query (only string query supported)", - requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @Schema(name = "full_text_query", requiredMode = Schema.RequiredMode.NOT_REQUIRED) @JsonProperty("full_text_query") - public StringFtsQuery getFullTextQuery() { + public QueryRequestFullTextQuery getFullTextQuery() { return fullTextQuery; } - public void setFullTextQuery(StringFtsQuery fullTextQuery) { + public void setFullTextQuery(QueryRequestFullTextQuery fullTextQuery) { this.fullTextQuery = fullTextQuery; } diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequestFullTextQuery.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequestFullTextQuery.java new file mode 100644 index 000000000..d4a8cec41 --- /dev/null +++ b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequestFullTextQuery.java @@ -0,0 +1,121 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.server.springboot.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.swagger.v3.oas.annotations.media.Schema; + +import javax.annotation.Generated; +import javax.validation.Valid; +import javax.validation.constraints.*; + +import java.util.*; +import java.util.Objects; + +/** Optional full-text search query. Provide either string_query or structured_query, not both. */ +@Schema( + name = "QueryRequest_full_text_query", + description = + "Optional full-text search query. Provide either string_query or structured_query, not both.") +@JsonTypeName("QueryRequest_full_text_query") +@Generated( + value = "org.openapitools.codegen.languages.SpringCodegen", + comments = "Generator version: 7.12.0") +public class QueryRequestFullTextQuery { + + private StringFtsQuery stringQuery; + + private StructuredFtsQuery structuredQuery; + + public QueryRequestFullTextQuery stringQuery(StringFtsQuery stringQuery) { + this.stringQuery = stringQuery; + return this; + } + + /** + * Get stringQuery + * + * @return stringQuery + */ + @Valid + @Schema(name = "string_query", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("string_query") + public StringFtsQuery getStringQuery() { + return stringQuery; + } + + public void setStringQuery(StringFtsQuery stringQuery) { + this.stringQuery = stringQuery; + } + + public QueryRequestFullTextQuery structuredQuery(StructuredFtsQuery structuredQuery) { + this.structuredQuery = structuredQuery; + return this; + } + + /** + * Get structuredQuery + * + * @return structuredQuery + */ + @Valid + @Schema(name = "structured_query", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("structured_query") + public StructuredFtsQuery getStructuredQuery() { + return structuredQuery; + } + + public void setStructuredQuery(StructuredFtsQuery structuredQuery) { + this.structuredQuery = structuredQuery; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + QueryRequestFullTextQuery queryRequestFullTextQuery = (QueryRequestFullTextQuery) o; + return Objects.equals(this.stringQuery, queryRequestFullTextQuery.stringQuery) + && Objects.equals(this.structuredQuery, queryRequestFullTextQuery.structuredQuery); + } + + @Override + public int hashCode() { + return Objects.hash(stringQuery, structuredQuery); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class QueryRequestFullTextQuery {\n"); + sb.append(" stringQuery: ").append(toIndentedString(stringQuery)).append("\n"); + sb.append(" structuredQuery: ").append(toIndentedString(structuredQuery)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} diff --git a/python/lance_namespace_urllib3_client/README.md b/python/lance_namespace_urllib3_client/README.md index 18639ddfd..74987d059 100644 --- a/python/lance_namespace_urllib3_client/README.md +++ b/python/lance_namespace_urllib3_client/README.md @@ -151,11 +151,6 @@ Class | Method | HTTP request | Description - [DropTableResponse](docs/DropTableResponse.md) - [ErrorResponse](docs/ErrorResponse.md) - [FtsQuery](docs/FtsQuery.md) - - [FtsQueryOneOf](docs/FtsQueryOneOf.md) - - [FtsQueryOneOf1](docs/FtsQueryOneOf1.md) - - [FtsQueryOneOf2](docs/FtsQueryOneOf2.md) - - [FtsQueryOneOf3](docs/FtsQueryOneOf3.md) - - [FtsQueryOneOf4](docs/FtsQueryOneOf4.md) - [IndexListItemResponse](docs/IndexListItemResponse.md) - [IndexListRequest](docs/IndexListRequest.md) - [IndexListResponse](docs/IndexListResponse.md) @@ -178,6 +173,7 @@ Class | Method | HTTP request | Description - [Operator](docs/Operator.md) - [PhraseQuery](docs/PhraseQuery.md) - [QueryRequest](docs/QueryRequest.md) + - [QueryRequestFullTextQuery](docs/QueryRequestFullTextQuery.md) - [RegisterTableRequest](docs/RegisterTableRequest.md) - [RegisterTableResponse](docs/RegisterTableResponse.md) - [SetPropertyMode](docs/SetPropertyMode.md) diff --git a/python/lance_namespace_urllib3_client/docs/BoostQuery.md b/python/lance_namespace_urllib3_client/docs/BoostQuery.md index 25cdc7fd5..41627f62e 100644 --- a/python/lance_namespace_urllib3_client/docs/BoostQuery.md +++ b/python/lance_namespace_urllib3_client/docs/BoostQuery.md @@ -1,13 +1,14 @@ # BoostQuery +Boost query that scores documents matching positive query higher and negative query lower ## Properties Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**negative** | **object** | | -**negative_boost** | **float** | | [optional] -**positive** | **object** | | +**positive** | [**FtsQuery**](FtsQuery.md) | | +**negative** | [**FtsQuery**](FtsQuery.md) | | +**negative_boost** | **float** | Boost factor for negative query (default: 0.5) | [optional] [default to 0.5] ## Example diff --git a/python/lance_namespace_urllib3_client/docs/FtsQuery.md b/python/lance_namespace_urllib3_client/docs/FtsQuery.md index a96f8022e..ef654343c 100644 --- a/python/lance_namespace_urllib3_client/docs/FtsQuery.md +++ b/python/lance_namespace_urllib3_client/docs/FtsQuery.md @@ -1,15 +1,16 @@ # FtsQuery +Full-text search query. Exactly one query type field must be provided. This structure follows the same pattern as AlterTransactionAction to minimize differences and compatibility issues across codegen in different languages. ## Properties Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**match** | [**MatchQuery**](MatchQuery.md) | | -**phrase** | [**PhraseQuery**](PhraseQuery.md) | | -**boost** | [**BoostQuery**](BoostQuery.md) | | -**multi_match** | [**MultiMatchQuery**](MultiMatchQuery.md) | | -**boolean** | [**BooleanQuery**](BooleanQuery.md) | | +**match** | [**MatchQuery**](MatchQuery.md) | | [optional] +**phrase** | [**PhraseQuery**](PhraseQuery.md) | | [optional] +**boost** | [**BoostQuery**](BoostQuery.md) | | [optional] +**multi_match** | [**MultiMatchQuery**](MultiMatchQuery.md) | | [optional] +**boolean** | [**BooleanQuery**](BooleanQuery.md) | | [optional] ## Example diff --git a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf.md b/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf.md deleted file mode 100644 index faacf98c3..000000000 --- a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf.md +++ /dev/null @@ -1,29 +0,0 @@ -# FtsQueryOneOf - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**match** | [**MatchQuery**](MatchQuery.md) | | - -## Example - -```python -from lance_namespace_urllib3_client.models.fts_query_one_of import FtsQueryOneOf - -# TODO update the JSON string below -json = "{}" -# create an instance of FtsQueryOneOf from a JSON string -fts_query_one_of_instance = FtsQueryOneOf.from_json(json) -# print the JSON string representation of the object -print(FtsQueryOneOf.to_json()) - -# convert the object into a dict -fts_query_one_of_dict = fts_query_one_of_instance.to_dict() -# create an instance of FtsQueryOneOf from a dict -fts_query_one_of_from_dict = FtsQueryOneOf.from_dict(fts_query_one_of_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf1.md b/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf1.md deleted file mode 100644 index 5a97c2b37..000000000 --- a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf1.md +++ /dev/null @@ -1,29 +0,0 @@ -# FtsQueryOneOf1 - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**phrase** | [**PhraseQuery**](PhraseQuery.md) | | - -## Example - -```python -from lance_namespace_urllib3_client.models.fts_query_one_of1 import FtsQueryOneOf1 - -# TODO update the JSON string below -json = "{}" -# create an instance of FtsQueryOneOf1 from a JSON string -fts_query_one_of1_instance = FtsQueryOneOf1.from_json(json) -# print the JSON string representation of the object -print(FtsQueryOneOf1.to_json()) - -# convert the object into a dict -fts_query_one_of1_dict = fts_query_one_of1_instance.to_dict() -# create an instance of FtsQueryOneOf1 from a dict -fts_query_one_of1_from_dict = FtsQueryOneOf1.from_dict(fts_query_one_of1_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf2.md b/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf2.md deleted file mode 100644 index 1491f572c..000000000 --- a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf2.md +++ /dev/null @@ -1,29 +0,0 @@ -# FtsQueryOneOf2 - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**boost** | [**BoostQuery**](BoostQuery.md) | | - -## Example - -```python -from lance_namespace_urllib3_client.models.fts_query_one_of2 import FtsQueryOneOf2 - -# TODO update the JSON string below -json = "{}" -# create an instance of FtsQueryOneOf2 from a JSON string -fts_query_one_of2_instance = FtsQueryOneOf2.from_json(json) -# print the JSON string representation of the object -print(FtsQueryOneOf2.to_json()) - -# convert the object into a dict -fts_query_one_of2_dict = fts_query_one_of2_instance.to_dict() -# create an instance of FtsQueryOneOf2 from a dict -fts_query_one_of2_from_dict = FtsQueryOneOf2.from_dict(fts_query_one_of2_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf3.md b/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf3.md deleted file mode 100644 index 1058fa1fc..000000000 --- a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf3.md +++ /dev/null @@ -1,29 +0,0 @@ -# FtsQueryOneOf3 - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**multi_match** | [**MultiMatchQuery**](MultiMatchQuery.md) | | - -## Example - -```python -from lance_namespace_urllib3_client.models.fts_query_one_of3 import FtsQueryOneOf3 - -# TODO update the JSON string below -json = "{}" -# create an instance of FtsQueryOneOf3 from a JSON string -fts_query_one_of3_instance = FtsQueryOneOf3.from_json(json) -# print the JSON string representation of the object -print(FtsQueryOneOf3.to_json()) - -# convert the object into a dict -fts_query_one_of3_dict = fts_query_one_of3_instance.to_dict() -# create an instance of FtsQueryOneOf3 from a dict -fts_query_one_of3_from_dict = FtsQueryOneOf3.from_dict(fts_query_one_of3_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf4.md b/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf4.md deleted file mode 100644 index 8348a45a0..000000000 --- a/python/lance_namespace_urllib3_client/docs/FtsQueryOneOf4.md +++ /dev/null @@ -1,29 +0,0 @@ -# FtsQueryOneOf4 - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**boolean** | [**BooleanQuery**](BooleanQuery.md) | | - -## Example - -```python -from lance_namespace_urllib3_client.models.fts_query_one_of4 import FtsQueryOneOf4 - -# TODO update the JSON string below -json = "{}" -# create an instance of FtsQueryOneOf4 from a JSON string -fts_query_one_of4_instance = FtsQueryOneOf4.from_json(json) -# print the JSON string representation of the object -print(FtsQueryOneOf4.to_json()) - -# convert the object into a dict -fts_query_one_of4_dict = fts_query_one_of4_instance.to_dict() -# create an instance of FtsQueryOneOf4 from a dict -fts_query_one_of4_from_dict = FtsQueryOneOf4.from_dict(fts_query_one_of4_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/python/lance_namespace_urllib3_client/docs/QueryRequest.md b/python/lance_namespace_urllib3_client/docs/QueryRequest.md index 83a629ea6..28706a3a7 100644 --- a/python/lance_namespace_urllib3_client/docs/QueryRequest.md +++ b/python/lance_namespace_urllib3_client/docs/QueryRequest.md @@ -13,7 +13,7 @@ Name | Type | Description | Notes **ef** | **int** | Search effort parameter for HNSW index | [optional] **fast_search** | **bool** | Whether to use fast search | [optional] **filter** | **str** | Optional SQL filter expression | [optional] -**full_text_query** | [**StringFtsQuery**](StringFtsQuery.md) | Optional full-text search query (only string query supported) | [optional] +**full_text_query** | [**QueryRequestFullTextQuery**](QueryRequestFullTextQuery.md) | | [optional] **k** | **int** | Number of results to return | **lower_bound** | **float** | Lower bound for search | [optional] **nprobes** | **int** | Number of probes for IVF index | [optional] diff --git a/python/lance_namespace_urllib3_client/docs/QueryRequestFullTextQuery.md b/python/lance_namespace_urllib3_client/docs/QueryRequestFullTextQuery.md new file mode 100644 index 000000000..b74434c61 --- /dev/null +++ b/python/lance_namespace_urllib3_client/docs/QueryRequestFullTextQuery.md @@ -0,0 +1,31 @@ +# QueryRequestFullTextQuery + +Optional full-text search query. Provide either string_query or structured_query, not both. + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**string_query** | [**StringFtsQuery**](StringFtsQuery.md) | | [optional] +**structured_query** | [**StructuredFtsQuery**](StructuredFtsQuery.md) | | [optional] + +## Example + +```python +from lance_namespace_urllib3_client.models.query_request_full_text_query import QueryRequestFullTextQuery + +# TODO update the JSON string below +json = "{}" +# create an instance of QueryRequestFullTextQuery from a JSON string +query_request_full_text_query_instance = QueryRequestFullTextQuery.from_json(json) +# print the JSON string representation of the object +print(QueryRequestFullTextQuery.to_json()) + +# convert the object into a dict +query_request_full_text_query_dict = query_request_full_text_query_instance.to_dict() +# create an instance of QueryRequestFullTextQuery from a dict +query_request_full_text_query_from_dict = QueryRequestFullTextQuery.from_dict(query_request_full_text_query_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/__init__.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/__init__.py index d6ba40eac..5ea95d36c 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/__init__.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/__init__.py @@ -65,11 +65,6 @@ from lance_namespace_urllib3_client.models.drop_table_response import DropTableResponse from lance_namespace_urllib3_client.models.error_response import ErrorResponse from lance_namespace_urllib3_client.models.fts_query import FtsQuery -from lance_namespace_urllib3_client.models.fts_query_one_of import FtsQueryOneOf -from lance_namespace_urllib3_client.models.fts_query_one_of1 import FtsQueryOneOf1 -from lance_namespace_urllib3_client.models.fts_query_one_of2 import FtsQueryOneOf2 -from lance_namespace_urllib3_client.models.fts_query_one_of3 import FtsQueryOneOf3 -from lance_namespace_urllib3_client.models.fts_query_one_of4 import FtsQueryOneOf4 from lance_namespace_urllib3_client.models.index_list_item_response import IndexListItemResponse from lance_namespace_urllib3_client.models.index_list_request import IndexListRequest from lance_namespace_urllib3_client.models.index_list_response import IndexListResponse @@ -92,6 +87,7 @@ from lance_namespace_urllib3_client.models.operator import Operator from lance_namespace_urllib3_client.models.phrase_query import PhraseQuery from lance_namespace_urllib3_client.models.query_request import QueryRequest +from lance_namespace_urllib3_client.models.query_request_full_text_query import QueryRequestFullTextQuery from lance_namespace_urllib3_client.models.register_table_request import RegisterTableRequest from lance_namespace_urllib3_client.models.register_table_response import RegisterTableResponse from lance_namespace_urllib3_client.models.set_property_mode import SetPropertyMode diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/__init__.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/__init__.py index c2b59dca2..b48f10ba9 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/__init__.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/__init__.py @@ -46,11 +46,6 @@ from lance_namespace_urllib3_client.models.drop_table_response import DropTableResponse from lance_namespace_urllib3_client.models.error_response import ErrorResponse from lance_namespace_urllib3_client.models.fts_query import FtsQuery -from lance_namespace_urllib3_client.models.fts_query_one_of import FtsQueryOneOf -from lance_namespace_urllib3_client.models.fts_query_one_of1 import FtsQueryOneOf1 -from lance_namespace_urllib3_client.models.fts_query_one_of2 import FtsQueryOneOf2 -from lance_namespace_urllib3_client.models.fts_query_one_of3 import FtsQueryOneOf3 -from lance_namespace_urllib3_client.models.fts_query_one_of4 import FtsQueryOneOf4 from lance_namespace_urllib3_client.models.index_list_item_response import IndexListItemResponse from lance_namespace_urllib3_client.models.index_list_request import IndexListRequest from lance_namespace_urllib3_client.models.index_list_response import IndexListResponse @@ -73,6 +68,7 @@ from lance_namespace_urllib3_client.models.operator import Operator from lance_namespace_urllib3_client.models.phrase_query import PhraseQuery from lance_namespace_urllib3_client.models.query_request import QueryRequest +from lance_namespace_urllib3_client.models.query_request_full_text_query import QueryRequestFullTextQuery from lance_namespace_urllib3_client.models.register_table_request import RegisterTableRequest from lance_namespace_urllib3_client.models.register_table_response import RegisterTableResponse from lance_namespace_urllib3_client.models.set_property_mode import SetPropertyMode diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/boost_query.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/boost_query.py index d9458c005..68bc6fb98 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/boost_query.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/boost_query.py @@ -17,19 +17,19 @@ import re # noqa: F401 import json -from pydantic import BaseModel, ConfigDict, StrictFloat, StrictInt +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt from typing import Any, ClassVar, Dict, List, Optional, Union from typing import Optional, Set from typing_extensions import Self class BoostQuery(BaseModel): """ - BoostQuery + Boost query that scores documents matching positive query higher and negative query lower """ # noqa: E501 - negative: Dict[str, Any] - negative_boost: Optional[Union[StrictFloat, StrictInt]] = None - positive: Dict[str, Any] - __properties: ClassVar[List[str]] = ["negative", "negative_boost", "positive"] + positive: FtsQuery + negative: FtsQuery + negative_boost: Optional[Union[StrictFloat, StrictInt]] = Field(default=0.5, description="Boost factor for negative query (default: 0.5)") + __properties: ClassVar[List[str]] = ["positive", "negative", "negative_boost"] model_config = ConfigDict( populate_by_name=True, @@ -70,6 +70,12 @@ def to_dict(self) -> Dict[str, Any]: exclude=excluded_fields, exclude_none=True, ) + # override the default output from pydantic by calling `to_dict()` of positive + if self.positive: + _dict['positive'] = self.positive.to_dict() + # override the default output from pydantic by calling `to_dict()` of negative + if self.negative: + _dict['negative'] = self.negative.to_dict() return _dict @classmethod @@ -82,10 +88,13 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: return cls.model_validate(obj) _obj = cls.model_validate({ - "negative": obj.get("negative"), - "negative_boost": obj.get("negative_boost"), - "positive": obj.get("positive") + "positive": FtsQuery.from_dict(obj["positive"]) if obj.get("positive") is not None else None, + "negative": FtsQuery.from_dict(obj["negative"]) if obj.get("negative") is not None else None, + "negative_boost": obj.get("negative_boost") if obj.get("negative_boost") is not None else 0.5 }) return _obj +from lance_namespace_urllib3_client.models.fts_query import FtsQuery +# TODO: Rewrite to not use raise_errors +BoostQuery.model_rebuild(raise_errors=False) diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query.py index 265f21624..d4f99d058 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query.py @@ -13,169 +13,105 @@ from __future__ import annotations -import json import pprint -from pydantic import BaseModel, ConfigDict, Field, StrictStr, ValidationError, field_validator -from typing import Any, List, Optional -from lance_namespace_urllib3_client.models.fts_query_one_of import FtsQueryOneOf -from lance_namespace_urllib3_client.models.fts_query_one_of1 import FtsQueryOneOf1 -from lance_namespace_urllib3_client.models.fts_query_one_of2 import FtsQueryOneOf2 -from lance_namespace_urllib3_client.models.fts_query_one_of3 import FtsQueryOneOf3 -from pydantic import StrictStr, Field -from typing import Union, List, Set, Optional, Dict -from typing_extensions import Literal, Self - -FTSQUERY_ONE_OF_SCHEMAS = ["FtsQueryOneOf", "FtsQueryOneOf1", "FtsQueryOneOf2", "FtsQueryOneOf3", "FtsQueryOneOf4"] +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict +from typing import Any, ClassVar, Dict, List, Optional +from lance_namespace_urllib3_client.models.match_query import MatchQuery +from lance_namespace_urllib3_client.models.multi_match_query import MultiMatchQuery +from lance_namespace_urllib3_client.models.phrase_query import PhraseQuery +from typing import Optional, Set +from typing_extensions import Self class FtsQuery(BaseModel): """ - FtsQuery - """ - # data type: FtsQueryOneOf - oneof_schema_1_validator: Optional[FtsQueryOneOf] = None - # data type: FtsQueryOneOf1 - oneof_schema_2_validator: Optional[FtsQueryOneOf1] = None - # data type: FtsQueryOneOf2 - oneof_schema_3_validator: Optional[FtsQueryOneOf2] = None - # data type: FtsQueryOneOf3 - oneof_schema_4_validator: Optional[FtsQueryOneOf3] = None - # data type: FtsQueryOneOf4 - oneof_schema_5_validator: Optional[FtsQueryOneOf4] = None - actual_instance: Optional[Union[FtsQueryOneOf, FtsQueryOneOf1, FtsQueryOneOf2, FtsQueryOneOf3, FtsQueryOneOf4]] = None - one_of_schemas: Set[str] = { "FtsQueryOneOf", "FtsQueryOneOf1", "FtsQueryOneOf2", "FtsQueryOneOf3", "FtsQueryOneOf4" } + Full-text search query. Exactly one query type field must be provided. This structure follows the same pattern as AlterTransactionAction to minimize differences and compatibility issues across codegen in different languages. + """ # noqa: E501 + match: Optional[MatchQuery] = None + phrase: Optional[PhraseQuery] = None + boost: Optional[BoostQuery] = None + multi_match: Optional[MultiMatchQuery] = None + boolean: Optional[BooleanQuery] = None + __properties: ClassVar[List[str]] = ["match", "phrase", "boost", "multi_match", "boolean"] model_config = ConfigDict( + populate_by_name=True, validate_assignment=True, protected_namespaces=(), ) - def __init__(self, *args, **kwargs) -> None: - if args: - if len(args) > 1: - raise ValueError("If a position argument is used, only 1 is allowed to set `actual_instance`") - if kwargs: - raise ValueError("If a position argument is used, keyword arguments cannot be used.") - super().__init__(actual_instance=args[0]) - else: - super().__init__(**kwargs) - - @field_validator('actual_instance') - def actual_instance_must_validate_oneof(cls, v): - instance = FtsQuery.model_construct() - error_messages = [] - match = 0 - # validate data type: FtsQueryOneOf - if not isinstance(v, FtsQueryOneOf): - error_messages.append(f"Error! Input type `{type(v)}` is not `FtsQueryOneOf`") - else: - match += 1 - # validate data type: FtsQueryOneOf1 - if not isinstance(v, FtsQueryOneOf1): - error_messages.append(f"Error! Input type `{type(v)}` is not `FtsQueryOneOf1`") - else: - match += 1 - # validate data type: FtsQueryOneOf2 - if not isinstance(v, FtsQueryOneOf2): - error_messages.append(f"Error! Input type `{type(v)}` is not `FtsQueryOneOf2`") - else: - match += 1 - # validate data type: FtsQueryOneOf3 - if not isinstance(v, FtsQueryOneOf3): - error_messages.append(f"Error! Input type `{type(v)}` is not `FtsQueryOneOf3`") - else: - match += 1 - # validate data type: FtsQueryOneOf4 - if not isinstance(v, FtsQueryOneOf4): - error_messages.append(f"Error! Input type `{type(v)}` is not `FtsQueryOneOf4`") - else: - match += 1 - if match > 1: - # more than 1 match - raise ValueError("Multiple matches found when setting `actual_instance` in FtsQuery with oneOf schemas: FtsQueryOneOf, FtsQueryOneOf1, FtsQueryOneOf2, FtsQueryOneOf3, FtsQueryOneOf4. Details: " + ", ".join(error_messages)) - elif match == 0: - # no match - raise ValueError("No match found when setting `actual_instance` in FtsQuery with oneOf schemas: FtsQueryOneOf, FtsQueryOneOf1, FtsQueryOneOf2, FtsQueryOneOf3, FtsQueryOneOf4. Details: " + ", ".join(error_messages)) - else: - return v + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) @classmethod - def from_dict(cls, obj: Union[str, Dict[str, Any]]) -> Self: - return cls.from_json(json.dumps(obj)) + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of FtsQuery from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of match + if self.match: + _dict['match'] = self.match.to_dict() + # override the default output from pydantic by calling `to_dict()` of phrase + if self.phrase: + _dict['phrase'] = self.phrase.to_dict() + # override the default output from pydantic by calling `to_dict()` of boost + if self.boost: + _dict['boost'] = self.boost.to_dict() + # override the default output from pydantic by calling `to_dict()` of multi_match + if self.multi_match: + _dict['multi_match'] = self.multi_match.to_dict() + # override the default output from pydantic by calling `to_dict()` of boolean + if self.boolean: + _dict['boolean'] = self.boolean.to_dict() + return _dict @classmethod - def from_json(cls, json_str: str) -> Self: - """Returns the object represented by the json string""" - instance = cls.model_construct() - error_messages = [] - match = 0 - - # deserialize data into FtsQueryOneOf - try: - instance.actual_instance = FtsQueryOneOf.from_json(json_str) - match += 1 - except (ValidationError, ValueError) as e: - error_messages.append(str(e)) - # deserialize data into FtsQueryOneOf1 - try: - instance.actual_instance = FtsQueryOneOf1.from_json(json_str) - match += 1 - except (ValidationError, ValueError) as e: - error_messages.append(str(e)) - # deserialize data into FtsQueryOneOf2 - try: - instance.actual_instance = FtsQueryOneOf2.from_json(json_str) - match += 1 - except (ValidationError, ValueError) as e: - error_messages.append(str(e)) - # deserialize data into FtsQueryOneOf3 - try: - instance.actual_instance = FtsQueryOneOf3.from_json(json_str) - match += 1 - except (ValidationError, ValueError) as e: - error_messages.append(str(e)) - # deserialize data into FtsQueryOneOf4 - try: - instance.actual_instance = FtsQueryOneOf4.from_json(json_str) - match += 1 - except (ValidationError, ValueError) as e: - error_messages.append(str(e)) - - if match > 1: - # more than 1 match - raise ValueError("Multiple matches found when deserializing the JSON string into FtsQuery with oneOf schemas: FtsQueryOneOf, FtsQueryOneOf1, FtsQueryOneOf2, FtsQueryOneOf3, FtsQueryOneOf4. Details: " + ", ".join(error_messages)) - elif match == 0: - # no match - raise ValueError("No match found when deserializing the JSON string into FtsQuery with oneOf schemas: FtsQueryOneOf, FtsQueryOneOf1, FtsQueryOneOf2, FtsQueryOneOf3, FtsQueryOneOf4. Details: " + ", ".join(error_messages)) - else: - return instance - - def to_json(self) -> str: - """Returns the JSON representation of the actual instance""" - if self.actual_instance is None: - return "null" - - if hasattr(self.actual_instance, "to_json") and callable(self.actual_instance.to_json): - return self.actual_instance.to_json() - else: - return json.dumps(self.actual_instance) - - def to_dict(self) -> Optional[Union[Dict[str, Any], FtsQueryOneOf, FtsQueryOneOf1, FtsQueryOneOf2, FtsQueryOneOf3, FtsQueryOneOf4]]: - """Returns the dict representation of the actual instance""" - if self.actual_instance is None: + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of FtsQuery from a dict""" + if obj is None: return None - if hasattr(self.actual_instance, "to_dict") and callable(self.actual_instance.to_dict): - return self.actual_instance.to_dict() - else: - # primitive type - return self.actual_instance + if not isinstance(obj, dict): + return cls.model_validate(obj) - def to_str(self) -> str: - """Returns the string representation of the actual instance""" - return pprint.pformat(self.model_dump()) + _obj = cls.model_validate({ + "match": MatchQuery.from_dict(obj["match"]) if obj.get("match") is not None else None, + "phrase": PhraseQuery.from_dict(obj["phrase"]) if obj.get("phrase") is not None else None, + "boost": BoostQuery.from_dict(obj["boost"]) if obj.get("boost") is not None else None, + "multi_match": MultiMatchQuery.from_dict(obj["multi_match"]) if obj.get("multi_match") is not None else None, + "boolean": BooleanQuery.from_dict(obj["boolean"]) if obj.get("boolean") is not None else None + }) + return _obj -from lance_namespace_urllib3_client.models.fts_query_one_of4 import FtsQueryOneOf4 +from lance_namespace_urllib3_client.models.boolean_query import BooleanQuery +from lance_namespace_urllib3_client.models.boost_query import BoostQuery # TODO: Rewrite to not use raise_errors FtsQuery.model_rebuild(raise_errors=False) diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of1.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of1.py deleted file mode 100644 index b6e5a0d68..000000000 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of1.py +++ /dev/null @@ -1,91 +0,0 @@ -# coding: utf-8 - -""" - Lance REST Namespace Specification - - This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - - The version of the OpenAPI document: 0.0.1 - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -from __future__ import annotations -import pprint -import re # noqa: F401 -import json - -from pydantic import BaseModel, ConfigDict -from typing import Any, ClassVar, Dict, List -from lance_namespace_urllib3_client.models.phrase_query import PhraseQuery -from typing import Optional, Set -from typing_extensions import Self - -class FtsQueryOneOf1(BaseModel): - """ - FtsQueryOneOf1 - """ # noqa: E501 - phrase: PhraseQuery - __properties: ClassVar[List[str]] = ["phrase"] - - model_config = ConfigDict( - populate_by_name=True, - validate_assignment=True, - protected_namespaces=(), - ) - - - def to_str(self) -> str: - """Returns the string representation of the model using alias""" - return pprint.pformat(self.model_dump(by_alias=True)) - - def to_json(self) -> str: - """Returns the JSON representation of the model using alias""" - # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead - return json.dumps(self.to_dict()) - - @classmethod - def from_json(cls, json_str: str) -> Optional[Self]: - """Create an instance of FtsQueryOneOf1 from a JSON string""" - return cls.from_dict(json.loads(json_str)) - - def to_dict(self) -> Dict[str, Any]: - """Return the dictionary representation of the model using alias. - - This has the following differences from calling pydantic's - `self.model_dump(by_alias=True)`: - - * `None` is only added to the output dict for nullable fields that - were set at model initialization. Other fields with value `None` - are ignored. - """ - excluded_fields: Set[str] = set([ - ]) - - _dict = self.model_dump( - by_alias=True, - exclude=excluded_fields, - exclude_none=True, - ) - # override the default output from pydantic by calling `to_dict()` of phrase - if self.phrase: - _dict['phrase'] = self.phrase.to_dict() - return _dict - - @classmethod - def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: - """Create an instance of FtsQueryOneOf1 from a dict""" - if obj is None: - return None - - if not isinstance(obj, dict): - return cls.model_validate(obj) - - _obj = cls.model_validate({ - "phrase": PhraseQuery.from_dict(obj["phrase"]) if obj.get("phrase") is not None else None - }) - return _obj - - diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of2.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of2.py deleted file mode 100644 index 589b48cdc..000000000 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of2.py +++ /dev/null @@ -1,91 +0,0 @@ -# coding: utf-8 - -""" - Lance REST Namespace Specification - - This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - - The version of the OpenAPI document: 0.0.1 - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -from __future__ import annotations -import pprint -import re # noqa: F401 -import json - -from pydantic import BaseModel, ConfigDict -from typing import Any, ClassVar, Dict, List -from lance_namespace_urllib3_client.models.boost_query import BoostQuery -from typing import Optional, Set -from typing_extensions import Self - -class FtsQueryOneOf2(BaseModel): - """ - FtsQueryOneOf2 - """ # noqa: E501 - boost: BoostQuery - __properties: ClassVar[List[str]] = ["boost"] - - model_config = ConfigDict( - populate_by_name=True, - validate_assignment=True, - protected_namespaces=(), - ) - - - def to_str(self) -> str: - """Returns the string representation of the model using alias""" - return pprint.pformat(self.model_dump(by_alias=True)) - - def to_json(self) -> str: - """Returns the JSON representation of the model using alias""" - # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead - return json.dumps(self.to_dict()) - - @classmethod - def from_json(cls, json_str: str) -> Optional[Self]: - """Create an instance of FtsQueryOneOf2 from a JSON string""" - return cls.from_dict(json.loads(json_str)) - - def to_dict(self) -> Dict[str, Any]: - """Return the dictionary representation of the model using alias. - - This has the following differences from calling pydantic's - `self.model_dump(by_alias=True)`: - - * `None` is only added to the output dict for nullable fields that - were set at model initialization. Other fields with value `None` - are ignored. - """ - excluded_fields: Set[str] = set([ - ]) - - _dict = self.model_dump( - by_alias=True, - exclude=excluded_fields, - exclude_none=True, - ) - # override the default output from pydantic by calling `to_dict()` of boost - if self.boost: - _dict['boost'] = self.boost.to_dict() - return _dict - - @classmethod - def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: - """Create an instance of FtsQueryOneOf2 from a dict""" - if obj is None: - return None - - if not isinstance(obj, dict): - return cls.model_validate(obj) - - _obj = cls.model_validate({ - "boost": BoostQuery.from_dict(obj["boost"]) if obj.get("boost") is not None else None - }) - return _obj - - diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of3.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of3.py deleted file mode 100644 index e964e438c..000000000 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of3.py +++ /dev/null @@ -1,91 +0,0 @@ -# coding: utf-8 - -""" - Lance REST Namespace Specification - - This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - - The version of the OpenAPI document: 0.0.1 - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -from __future__ import annotations -import pprint -import re # noqa: F401 -import json - -from pydantic import BaseModel, ConfigDict -from typing import Any, ClassVar, Dict, List -from lance_namespace_urllib3_client.models.multi_match_query import MultiMatchQuery -from typing import Optional, Set -from typing_extensions import Self - -class FtsQueryOneOf3(BaseModel): - """ - FtsQueryOneOf3 - """ # noqa: E501 - multi_match: MultiMatchQuery - __properties: ClassVar[List[str]] = ["multi_match"] - - model_config = ConfigDict( - populate_by_name=True, - validate_assignment=True, - protected_namespaces=(), - ) - - - def to_str(self) -> str: - """Returns the string representation of the model using alias""" - return pprint.pformat(self.model_dump(by_alias=True)) - - def to_json(self) -> str: - """Returns the JSON representation of the model using alias""" - # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead - return json.dumps(self.to_dict()) - - @classmethod - def from_json(cls, json_str: str) -> Optional[Self]: - """Create an instance of FtsQueryOneOf3 from a JSON string""" - return cls.from_dict(json.loads(json_str)) - - def to_dict(self) -> Dict[str, Any]: - """Return the dictionary representation of the model using alias. - - This has the following differences from calling pydantic's - `self.model_dump(by_alias=True)`: - - * `None` is only added to the output dict for nullable fields that - were set at model initialization. Other fields with value `None` - are ignored. - """ - excluded_fields: Set[str] = set([ - ]) - - _dict = self.model_dump( - by_alias=True, - exclude=excluded_fields, - exclude_none=True, - ) - # override the default output from pydantic by calling `to_dict()` of multi_match - if self.multi_match: - _dict['multi_match'] = self.multi_match.to_dict() - return _dict - - @classmethod - def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: - """Create an instance of FtsQueryOneOf3 from a dict""" - if obj is None: - return None - - if not isinstance(obj, dict): - return cls.model_validate(obj) - - _obj = cls.model_validate({ - "multi_match": MultiMatchQuery.from_dict(obj["multi_match"]) if obj.get("multi_match") is not None else None - }) - return _obj - - diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of4.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of4.py deleted file mode 100644 index 7368eae89..000000000 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of4.py +++ /dev/null @@ -1,93 +0,0 @@ -# coding: utf-8 - -""" - Lance REST Namespace Specification - - This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - - The version of the OpenAPI document: 0.0.1 - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -from __future__ import annotations -import pprint -import re # noqa: F401 -import json - -from pydantic import BaseModel, ConfigDict -from typing import Any, ClassVar, Dict, List -from typing import Optional, Set -from typing_extensions import Self - -class FtsQueryOneOf4(BaseModel): - """ - FtsQueryOneOf4 - """ # noqa: E501 - boolean: BooleanQuery - __properties: ClassVar[List[str]] = ["boolean"] - - model_config = ConfigDict( - populate_by_name=True, - validate_assignment=True, - protected_namespaces=(), - ) - - - def to_str(self) -> str: - """Returns the string representation of the model using alias""" - return pprint.pformat(self.model_dump(by_alias=True)) - - def to_json(self) -> str: - """Returns the JSON representation of the model using alias""" - # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead - return json.dumps(self.to_dict()) - - @classmethod - def from_json(cls, json_str: str) -> Optional[Self]: - """Create an instance of FtsQueryOneOf4 from a JSON string""" - return cls.from_dict(json.loads(json_str)) - - def to_dict(self) -> Dict[str, Any]: - """Return the dictionary representation of the model using alias. - - This has the following differences from calling pydantic's - `self.model_dump(by_alias=True)`: - - * `None` is only added to the output dict for nullable fields that - were set at model initialization. Other fields with value `None` - are ignored. - """ - excluded_fields: Set[str] = set([ - ]) - - _dict = self.model_dump( - by_alias=True, - exclude=excluded_fields, - exclude_none=True, - ) - # override the default output from pydantic by calling `to_dict()` of boolean - if self.boolean: - _dict['boolean'] = self.boolean.to_dict() - return _dict - - @classmethod - def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: - """Create an instance of FtsQueryOneOf4 from a dict""" - if obj is None: - return None - - if not isinstance(obj, dict): - return cls.model_validate(obj) - - _obj = cls.model_validate({ - "boolean": BooleanQuery.from_dict(obj["boolean"]) if obj.get("boolean") is not None else None - }) - return _obj - -from lance_namespace_urllib3_client.models.boolean_query import BooleanQuery -# TODO: Rewrite to not use raise_errors -FtsQueryOneOf4.model_rebuild(raise_errors=False) - diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request.py index 96e3f00e9..1e9347f7c 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request.py @@ -20,7 +20,7 @@ from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictFloat, StrictInt, StrictStr from typing import Any, ClassVar, Dict, List, Optional, Union from typing_extensions import Annotated -from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery +from lance_namespace_urllib3_client.models.query_request_full_text_query import QueryRequestFullTextQuery from typing import Optional, Set from typing_extensions import Self @@ -36,7 +36,7 @@ class QueryRequest(BaseModel): ef: Optional[Annotated[int, Field(strict=True, ge=0)]] = Field(default=None, description="Search effort parameter for HNSW index") fast_search: Optional[StrictBool] = Field(default=None, description="Whether to use fast search") filter: Optional[StrictStr] = Field(default=None, description="Optional SQL filter expression") - full_text_query: Optional[StringFtsQuery] = Field(default=None, description="Optional full-text search query (only string query supported)") + full_text_query: Optional[QueryRequestFullTextQuery] = None k: Annotated[int, Field(strict=True, ge=0)] = Field(description="Number of results to return") lower_bound: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="Lower bound for search") nprobes: Optional[Annotated[int, Field(strict=True, ge=0)]] = Field(default=None, description="Number of probes for IVF index") @@ -182,7 +182,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "ef": obj.get("ef"), "fast_search": obj.get("fast_search"), "filter": obj.get("filter"), - "full_text_query": StringFtsQuery.from_dict(obj["full_text_query"]) if obj.get("full_text_query") is not None else None, + "full_text_query": QueryRequestFullTextQuery.from_dict(obj["full_text_query"]) if obj.get("full_text_query") is not None else None, "k": obj.get("k"), "lower_bound": obj.get("lower_bound"), "nprobes": obj.get("nprobes"), diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request_full_text_query.py similarity index 67% rename from python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of.py rename to python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request_full_text_query.py index 7cc332925..5e814bcfd 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/fts_query_one_of.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request_full_text_query.py @@ -18,17 +18,19 @@ import json from pydantic import BaseModel, ConfigDict -from typing import Any, ClassVar, Dict, List -from lance_namespace_urllib3_client.models.match_query import MatchQuery +from typing import Any, ClassVar, Dict, List, Optional +from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery +from lance_namespace_urllib3_client.models.structured_fts_query import StructuredFtsQuery from typing import Optional, Set from typing_extensions import Self -class FtsQueryOneOf(BaseModel): +class QueryRequestFullTextQuery(BaseModel): """ - FtsQueryOneOf + Optional full-text search query. Provide either string_query or structured_query, not both. """ # noqa: E501 - match: MatchQuery - __properties: ClassVar[List[str]] = ["match"] + string_query: Optional[StringFtsQuery] = None + structured_query: Optional[StructuredFtsQuery] = None + __properties: ClassVar[List[str]] = ["string_query", "structured_query"] model_config = ConfigDict( populate_by_name=True, @@ -48,7 +50,7 @@ def to_json(self) -> str: @classmethod def from_json(cls, json_str: str) -> Optional[Self]: - """Create an instance of FtsQueryOneOf from a JSON string""" + """Create an instance of QueryRequestFullTextQuery from a JSON string""" return cls.from_dict(json.loads(json_str)) def to_dict(self) -> Dict[str, Any]: @@ -69,14 +71,17 @@ def to_dict(self) -> Dict[str, Any]: exclude=excluded_fields, exclude_none=True, ) - # override the default output from pydantic by calling `to_dict()` of match - if self.match: - _dict['match'] = self.match.to_dict() + # override the default output from pydantic by calling `to_dict()` of string_query + if self.string_query: + _dict['string_query'] = self.string_query.to_dict() + # override the default output from pydantic by calling `to_dict()` of structured_query + if self.structured_query: + _dict['structured_query'] = self.structured_query.to_dict() return _dict @classmethod def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: - """Create an instance of FtsQueryOneOf from a dict""" + """Create an instance of QueryRequestFullTextQuery from a dict""" if obj is None: return None @@ -84,7 +89,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: return cls.model_validate(obj) _obj = cls.model_validate({ - "match": MatchQuery.from_dict(obj["match"]) if obj.get("match") is not None else None + "string_query": StringFtsQuery.from_dict(obj["string_query"]) if obj.get("string_query") is not None else None, + "structured_query": StructuredFtsQuery.from_dict(obj["structured_query"]) if obj.get("structured_query") is not None else None }) return _obj diff --git a/python/lance_namespace_urllib3_client/test/test_boolean_query.py b/python/lance_namespace_urllib3_client/test/test_boolean_query.py index b4be741b3..8e16c26b1 100644 --- a/python/lance_namespace_urllib3_client/test/test_boolean_query.py +++ b/python/lance_namespace_urllib3_client/test/test_boolean_query.py @@ -36,25 +36,307 @@ def make_instance(self, include_optional) -> BooleanQuery: if include_optional: return BooleanQuery( must = [ - null + lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = , + must_not = [ + + ], + should = [ + + ], ), ) ], must_not = [ - null + lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = , + should = [ + + ], ), ) ], should = [ - null + lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = , ), ) ] ) else: return BooleanQuery( must = [ - null + lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = , + must_not = [ + + ], + should = [ + + ], ), ) ], must_not = [ - null + lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = , + should = [ + + ], ), ) ], should = [ - null + lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = , ), ) ], ) """ diff --git a/python/lance_namespace_urllib3_client/test/test_boost_query.py b/python/lance_namespace_urllib3_client/test/test_boost_query.py index 3838000e7..8b6232d9d 100644 --- a/python/lance_namespace_urllib3_client/test/test_boost_query.py +++ b/python/lance_namespace_urllib3_client/test/test_boost_query.py @@ -35,14 +35,210 @@ def make_instance(self, include_optional) -> BoostQuery: model = BoostQuery() if include_optional: return BoostQuery( - negative = lance_namespace_urllib3_client.models.negative.negative(), - negative_boost = 1.337, - positive = lance_namespace_urllib3_client.models.positive.positive() + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative_boost = 1.337 ) else: return BoostQuery( - negative = lance_namespace_urllib3_client.models.negative.negative(), - positive = lance_namespace_urllib3_client.models.positive.positive(), + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), ) """ diff --git a/python/lance_namespace_urllib3_client/test/test_fts_query.py b/python/lance_namespace_urllib3_client/test/test_fts_query.py index aec031949..2c7afa4c2 100644 --- a/python/lance_namespace_urllib3_client/test/test_fts_query.py +++ b/python/lance_namespace_urllib3_client/test/test_fts_query.py @@ -48,49 +48,39 @@ def make_instance(self, include_optional) -> FtsQuery: slop = 0, terms = '', ), boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( - negative = lance_namespace_urllib3_client.models.negative.negative(), - negative_boost = 1.337, - positive = lance_namespace_urllib3_client.models.positive.positive(), ), - multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( - match_queries = [ - lance_namespace_urllib3_client.models.match_query.MatchQuery( - boost = 1.337, + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( column = '', fuzziness = 0, max_expansions = 0, operator = 'And', prefix_length = 0, - terms = '', ) - ], ), - boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( - must = [ - null - ], - must_not = [ - null - ], - should = [ - null - ], ) - ) - else: - return FtsQuery( - match = lance_namespace_urllib3_client.models.match_query.MatchQuery( - boost = 1.337, - column = '', - fuzziness = 0, - max_expansions = 0, - operator = 'And', - prefix_length = 0, - terms = '', ), - phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( - column = '', - slop = 0, - terms = '', ), - boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( - negative = lance_namespace_urllib3_client.models.negative.negative(), - negative_boost = 1.337, - positive = lance_namespace_urllib3_client.models.positive.positive(), ), + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + lance_namespace_urllib3_client.models.fts_query.FtsQuery() + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( match_queries = [ lance_namespace_urllib3_client.models.match_query.MatchQuery( @@ -104,14 +94,46 @@ def make_instance(self, include_optional) -> FtsQuery: ], ), boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( must = [ - null + lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), ) ], must_not = [ - null + ], should = [ - null - ], ), + + ], ) + ) + else: + return FtsQuery( ) """ diff --git a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of.py b/python/lance_namespace_urllib3_client/test/test_fts_query_one_of.py deleted file mode 100644 index de83a4b23..000000000 --- a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of.py +++ /dev/null @@ -1,66 +0,0 @@ -# coding: utf-8 - -""" - Lance REST Namespace Specification - - This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - - The version of the OpenAPI document: 0.0.1 - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -import unittest - -from lance_namespace_urllib3_client.models.fts_query_one_of import FtsQueryOneOf - -class TestFtsQueryOneOf(unittest.TestCase): - """FtsQueryOneOf unit test stubs""" - - def setUp(self): - pass - - def tearDown(self): - pass - - def make_instance(self, include_optional) -> FtsQueryOneOf: - """Test FtsQueryOneOf - include_optional is a boolean, when False only required - params are included, when True both required and - optional params are included """ - # uncomment below to create an instance of `FtsQueryOneOf` - """ - model = FtsQueryOneOf() - if include_optional: - return FtsQueryOneOf( - match = lance_namespace_urllib3_client.models.match_query.MatchQuery( - boost = 1.337, - column = '', - fuzziness = 0, - max_expansions = 0, - operator = 'And', - prefix_length = 0, - terms = '', ) - ) - else: - return FtsQueryOneOf( - match = lance_namespace_urllib3_client.models.match_query.MatchQuery( - boost = 1.337, - column = '', - fuzziness = 0, - max_expansions = 0, - operator = 'And', - prefix_length = 0, - terms = '', ), - ) - """ - - def testFtsQueryOneOf(self): - """Test FtsQueryOneOf""" - # inst_req_only = self.make_instance(include_optional=False) - # inst_req_and_optional = self.make_instance(include_optional=True) - -if __name__ == '__main__': - unittest.main() diff --git a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of1.py b/python/lance_namespace_urllib3_client/test/test_fts_query_one_of1.py deleted file mode 100644 index 192b5f27e..000000000 --- a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of1.py +++ /dev/null @@ -1,58 +0,0 @@ -# coding: utf-8 - -""" - Lance REST Namespace Specification - - This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - - The version of the OpenAPI document: 0.0.1 - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -import unittest - -from lance_namespace_urllib3_client.models.fts_query_one_of1 import FtsQueryOneOf1 - -class TestFtsQueryOneOf1(unittest.TestCase): - """FtsQueryOneOf1 unit test stubs""" - - def setUp(self): - pass - - def tearDown(self): - pass - - def make_instance(self, include_optional) -> FtsQueryOneOf1: - """Test FtsQueryOneOf1 - include_optional is a boolean, when False only required - params are included, when True both required and - optional params are included """ - # uncomment below to create an instance of `FtsQueryOneOf1` - """ - model = FtsQueryOneOf1() - if include_optional: - return FtsQueryOneOf1( - phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( - column = '', - slop = 0, - terms = '', ) - ) - else: - return FtsQueryOneOf1( - phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( - column = '', - slop = 0, - terms = '', ), - ) - """ - - def testFtsQueryOneOf1(self): - """Test FtsQueryOneOf1""" - # inst_req_only = self.make_instance(include_optional=False) - # inst_req_and_optional = self.make_instance(include_optional=True) - -if __name__ == '__main__': - unittest.main() diff --git a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of2.py b/python/lance_namespace_urllib3_client/test/test_fts_query_one_of2.py deleted file mode 100644 index 49c4663e0..000000000 --- a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of2.py +++ /dev/null @@ -1,58 +0,0 @@ -# coding: utf-8 - -""" - Lance REST Namespace Specification - - This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - - The version of the OpenAPI document: 0.0.1 - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -import unittest - -from lance_namespace_urllib3_client.models.fts_query_one_of2 import FtsQueryOneOf2 - -class TestFtsQueryOneOf2(unittest.TestCase): - """FtsQueryOneOf2 unit test stubs""" - - def setUp(self): - pass - - def tearDown(self): - pass - - def make_instance(self, include_optional) -> FtsQueryOneOf2: - """Test FtsQueryOneOf2 - include_optional is a boolean, when False only required - params are included, when True both required and - optional params are included """ - # uncomment below to create an instance of `FtsQueryOneOf2` - """ - model = FtsQueryOneOf2() - if include_optional: - return FtsQueryOneOf2( - boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( - negative = lance_namespace_urllib3_client.models.negative.negative(), - negative_boost = 1.337, - positive = lance_namespace_urllib3_client.models.positive.positive(), ) - ) - else: - return FtsQueryOneOf2( - boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( - negative = lance_namespace_urllib3_client.models.negative.negative(), - negative_boost = 1.337, - positive = lance_namespace_urllib3_client.models.positive.positive(), ), - ) - """ - - def testFtsQueryOneOf2(self): - """Test FtsQueryOneOf2""" - # inst_req_only = self.make_instance(include_optional=False) - # inst_req_and_optional = self.make_instance(include_optional=True) - -if __name__ == '__main__': - unittest.main() diff --git a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of3.py b/python/lance_namespace_urllib3_client/test/test_fts_query_one_of3.py deleted file mode 100644 index 8162cbc5f..000000000 --- a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of3.py +++ /dev/null @@ -1,72 +0,0 @@ -# coding: utf-8 - -""" - Lance REST Namespace Specification - - This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - - The version of the OpenAPI document: 0.0.1 - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -import unittest - -from lance_namespace_urllib3_client.models.fts_query_one_of3 import FtsQueryOneOf3 - -class TestFtsQueryOneOf3(unittest.TestCase): - """FtsQueryOneOf3 unit test stubs""" - - def setUp(self): - pass - - def tearDown(self): - pass - - def make_instance(self, include_optional) -> FtsQueryOneOf3: - """Test FtsQueryOneOf3 - include_optional is a boolean, when False only required - params are included, when True both required and - optional params are included """ - # uncomment below to create an instance of `FtsQueryOneOf3` - """ - model = FtsQueryOneOf3() - if include_optional: - return FtsQueryOneOf3( - multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( - match_queries = [ - lance_namespace_urllib3_client.models.match_query.MatchQuery( - boost = 1.337, - column = '', - fuzziness = 0, - max_expansions = 0, - operator = 'And', - prefix_length = 0, - terms = '', ) - ], ) - ) - else: - return FtsQueryOneOf3( - multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( - match_queries = [ - lance_namespace_urllib3_client.models.match_query.MatchQuery( - boost = 1.337, - column = '', - fuzziness = 0, - max_expansions = 0, - operator = 'And', - prefix_length = 0, - terms = '', ) - ], ), - ) - """ - - def testFtsQueryOneOf3(self): - """Test FtsQueryOneOf3""" - # inst_req_only = self.make_instance(include_optional=False) - # inst_req_and_optional = self.make_instance(include_optional=True) - -if __name__ == '__main__': - unittest.main() diff --git a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of4.py b/python/lance_namespace_urllib3_client/test/test_fts_query_one_of4.py deleted file mode 100644 index 2c263e7f6..000000000 --- a/python/lance_namespace_urllib3_client/test/test_fts_query_one_of4.py +++ /dev/null @@ -1,70 +0,0 @@ -# coding: utf-8 - -""" - Lance REST Namespace Specification - - This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - - The version of the OpenAPI document: 0.0.1 - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -import unittest - -from lance_namespace_urllib3_client.models.fts_query_one_of4 import FtsQueryOneOf4 - -class TestFtsQueryOneOf4(unittest.TestCase): - """FtsQueryOneOf4 unit test stubs""" - - def setUp(self): - pass - - def tearDown(self): - pass - - def make_instance(self, include_optional) -> FtsQueryOneOf4: - """Test FtsQueryOneOf4 - include_optional is a boolean, when False only required - params are included, when True both required and - optional params are included """ - # uncomment below to create an instance of `FtsQueryOneOf4` - """ - model = FtsQueryOneOf4() - if include_optional: - return FtsQueryOneOf4( - boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( - must = [ - null - ], - must_not = [ - null - ], - should = [ - null - ], ) - ) - else: - return FtsQueryOneOf4( - boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( - must = [ - null - ], - must_not = [ - null - ], - should = [ - null - ], ), - ) - """ - - def testFtsQueryOneOf4(self): - """Test FtsQueryOneOf4""" - # inst_req_only = self.make_instance(include_optional=False) - # inst_req_and_optional = self.make_instance(include_optional=True) - -if __name__ == '__main__': - unittest.main() diff --git a/python/lance_namespace_urllib3_client/test/test_query_request.py b/python/lance_namespace_urllib3_client/test/test_query_request.py index 9a20af511..bfcf5ea07 100644 --- a/python/lance_namespace_urllib3_client/test/test_query_request.py +++ b/python/lance_namespace_urllib3_client/test/test_query_request.py @@ -47,11 +47,63 @@ def make_instance(self, include_optional) -> QueryRequest: ef = 0, fast_search = True, filter = '', - full_text_query = lance_namespace_urllib3_client.models.string_fts_query.StringFtsQuery( - columns = [ - '' - ], - query = '', ), + full_text_query = lance_namespace_urllib3_client.models.query_request_full_text_query.QueryRequest_full_text_query( + string_query = lance_namespace_urllib3_client.models.string_fts_query.StringFtsQuery( + columns = [ + '' + ], + query = '', ), + structured_query = lance_namespace_urllib3_client.models.structured_fts_query.StructuredFtsQuery( + query = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), ), ), k = 0, lower_bound = 1.337, nprobes = 0, diff --git a/python/lance_namespace_urllib3_client/test/test_query_request_full_text_query.py b/python/lance_namespace_urllib3_client/test/test_query_request_full_text_query.py new file mode 100644 index 000000000..1c0818ad8 --- /dev/null +++ b/python/lance_namespace_urllib3_client/test/test_query_request_full_text_query.py @@ -0,0 +1,106 @@ +# coding: utf-8 + +""" + Lance REST Namespace Specification + + This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. + + The version of the OpenAPI document: 0.0.1 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from lance_namespace_urllib3_client.models.query_request_full_text_query import QueryRequestFullTextQuery + +class TestQueryRequestFullTextQuery(unittest.TestCase): + """QueryRequestFullTextQuery unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> QueryRequestFullTextQuery: + """Test QueryRequestFullTextQuery + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `QueryRequestFullTextQuery` + """ + model = QueryRequestFullTextQuery() + if include_optional: + return QueryRequestFullTextQuery( + string_query = lance_namespace_urllib3_client.models.string_fts_query.StringFtsQuery( + columns = [ + '' + ], + query = '', ), + structured_query = lance_namespace_urllib3_client.models.structured_fts_query.StructuredFtsQuery( + query = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), ) + ) + else: + return QueryRequestFullTextQuery( + ) + """ + + def testQueryRequestFullTextQuery(self): + """Test QueryRequestFullTextQuery""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/python/lance_namespace_urllib3_client/test/test_structured_fts_query.py b/python/lance_namespace_urllib3_client/test/test_structured_fts_query.py index b87a74e0d..d37ed0528 100644 --- a/python/lance_namespace_urllib3_client/test/test_structured_fts_query.py +++ b/python/lance_namespace_urllib3_client/test/test_structured_fts_query.py @@ -35,11 +35,109 @@ def make_instance(self, include_optional) -> StructuredFtsQuery: model = StructuredFtsQuery() if include_optional: return StructuredFtsQuery( - query = None + query = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ) ) else: return StructuredFtsQuery( - query = None, + query = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + match = lance_namespace_urllib3_client.models.match_query.MatchQuery( + boost = 1.337, + column = '', + fuzziness = 0, + max_expansions = 0, + operator = 'And', + prefix_length = 0, + terms = '', ), + phrase = lance_namespace_urllib3_client.models.phrase_query.PhraseQuery( + column = '', + slop = 0, + terms = '', ), + boost = lance_namespace_urllib3_client.models.boost_query.BoostQuery( + positive = lance_namespace_urllib3_client.models.fts_query.FtsQuery( + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + lance_namespace_urllib3_client.models.match_query.MatchQuery( + column = '', + fuzziness = 0, + max_expansions = 0, + prefix_length = 0, + terms = '', ) + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), + negative = , + negative_boost = 1.337, ), + multi_match = lance_namespace_urllib3_client.models.multi_match_query.MultiMatchQuery( + match_queries = [ + + ], ), + boolean = lance_namespace_urllib3_client.models.boolean_query.BooleanQuery( + must = [ + + ], + must_not = [ + + ], + should = [ + + ], ), ), ) """ diff --git a/rust/lance-namespace-reqwest-client/README.md b/rust/lance-namespace-reqwest-client/README.md index 1fa8955d5..760af1051 100644 --- a/rust/lance-namespace-reqwest-client/README.md +++ b/rust/lance-namespace-reqwest-client/README.md @@ -90,11 +90,6 @@ Class | Method | HTTP request | Description - [DropTableResponse](docs/DropTableResponse.md) - [ErrorResponse](docs/ErrorResponse.md) - [FtsQuery](docs/FtsQuery.md) - - [FtsQueryOneOf](docs/FtsQueryOneOf.md) - - [FtsQueryOneOf1](docs/FtsQueryOneOf1.md) - - [FtsQueryOneOf2](docs/FtsQueryOneOf2.md) - - [FtsQueryOneOf3](docs/FtsQueryOneOf3.md) - - [FtsQueryOneOf4](docs/FtsQueryOneOf4.md) - [IndexListItemResponse](docs/IndexListItemResponse.md) - [IndexListRequest](docs/IndexListRequest.md) - [IndexListResponse](docs/IndexListResponse.md) @@ -117,6 +112,7 @@ Class | Method | HTTP request | Description - [Operator](docs/Operator.md) - [PhraseQuery](docs/PhraseQuery.md) - [QueryRequest](docs/QueryRequest.md) + - [QueryRequestFullTextQuery](docs/QueryRequestFullTextQuery.md) - [RegisterTableRequest](docs/RegisterTableRequest.md) - [RegisterTableResponse](docs/RegisterTableResponse.md) - [SetPropertyMode](docs/SetPropertyMode.md) diff --git a/rust/lance-namespace-reqwest-client/docs/BoostQuery.md b/rust/lance-namespace-reqwest-client/docs/BoostQuery.md index 5c5f09916..2650a87d1 100644 --- a/rust/lance-namespace-reqwest-client/docs/BoostQuery.md +++ b/rust/lance-namespace-reqwest-client/docs/BoostQuery.md @@ -4,9 +4,9 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**negative** | [**serde_json::Value**](.md) | | -**negative_boost** | Option<**f32**> | | [optional] -**positive** | [**serde_json::Value**](.md) | | +**positive** | [**models::FtsQuery**](FtsQuery.md) | | +**negative** | [**models::FtsQuery**](FtsQuery.md) | | +**negative_boost** | Option<**f32**> | Boost factor for negative query (default: 0.5) | [optional][default to 0.5] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/rust/lance-namespace-reqwest-client/docs/FtsQuery.md b/rust/lance-namespace-reqwest-client/docs/FtsQuery.md index 4e72368b5..7d35f3333 100644 --- a/rust/lance-namespace-reqwest-client/docs/FtsQuery.md +++ b/rust/lance-namespace-reqwest-client/docs/FtsQuery.md @@ -1,14 +1,14 @@ # FtsQuery -## Enum Variants - -| Name | Description | -|---- | -----| -| FtsQueryOneOf | | -| FtsQueryOneOf1 | | -| FtsQueryOneOf2 | | -| FtsQueryOneOf3 | | -| FtsQueryOneOf4 | | +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**r#match** | Option<[**models::MatchQuery**](MatchQuery.md)> | | [optional] +**phrase** | Option<[**models::PhraseQuery**](PhraseQuery.md)> | | [optional] +**boost** | Option<[**models::BoostQuery**](BoostQuery.md)> | | [optional] +**multi_match** | Option<[**models::MultiMatchQuery**](MultiMatchQuery.md)> | | [optional] +**boolean** | Option<[**models::BooleanQuery**](BooleanQuery.md)> | | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf.md b/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf.md deleted file mode 100644 index c298a3db7..000000000 --- a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf.md +++ /dev/null @@ -1,11 +0,0 @@ -# FtsQueryOneOf - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**r#match** | [**models::MatchQuery**](MatchQuery.md) | | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf1.md b/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf1.md deleted file mode 100644 index fbcba8296..000000000 --- a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf1.md +++ /dev/null @@ -1,11 +0,0 @@ -# FtsQueryOneOf1 - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**phrase** | [**models::PhraseQuery**](PhraseQuery.md) | | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf2.md b/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf2.md deleted file mode 100644 index 67688272f..000000000 --- a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf2.md +++ /dev/null @@ -1,11 +0,0 @@ -# FtsQueryOneOf2 - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**boost** | [**models::BoostQuery**](BoostQuery.md) | | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf4.md b/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf4.md deleted file mode 100644 index 318447314..000000000 --- a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf4.md +++ /dev/null @@ -1,11 +0,0 @@ -# FtsQueryOneOf4 - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**boolean** | [**models::BooleanQuery**](BooleanQuery.md) | | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/rust/lance-namespace-reqwest-client/docs/QueryRequest.md b/rust/lance-namespace-reqwest-client/docs/QueryRequest.md index 8ac6633d6..d7160cd94 100644 --- a/rust/lance-namespace-reqwest-client/docs/QueryRequest.md +++ b/rust/lance-namespace-reqwest-client/docs/QueryRequest.md @@ -12,7 +12,7 @@ Name | Type | Description | Notes **ef** | Option<**i32**> | Search effort parameter for HNSW index | [optional] **fast_search** | Option<**bool**> | Whether to use fast search | [optional] **filter** | Option<**String**> | Optional SQL filter expression | [optional] -**full_text_query** | Option<[**models::StringFtsQuery**](StringFtsQuery.md)> | Optional full-text search query (only string query supported) | [optional] +**full_text_query** | Option<[**models::QueryRequestFullTextQuery**](QueryRequest_full_text_query.md)> | | [optional] **k** | **i32** | Number of results to return | **lower_bound** | Option<**f32**> | Lower bound for search | [optional] **nprobes** | Option<**i32**> | Number of probes for IVF index | [optional] diff --git a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf3.md b/rust/lance-namespace-reqwest-client/docs/QueryRequestFullTextQuery.md similarity index 55% rename from rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf3.md rename to rust/lance-namespace-reqwest-client/docs/QueryRequestFullTextQuery.md index 85eaa3096..3600cc846 100644 --- a/rust/lance-namespace-reqwest-client/docs/FtsQueryOneOf3.md +++ b/rust/lance-namespace-reqwest-client/docs/QueryRequestFullTextQuery.md @@ -1,10 +1,11 @@ -# FtsQueryOneOf3 +# QueryRequestFullTextQuery ## Properties Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**multi_match** | [**models::MultiMatchQuery**](MultiMatchQuery.md) | | +**string_query** | Option<[**models::StringFtsQuery**](StringFtsQuery.md)> | | [optional] +**structured_query** | Option<[**models::StructuredFtsQuery**](StructuredFtsQuery.md)> | | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/rust/lance-namespace-reqwest-client/src/models/boost_query.rs b/rust/lance-namespace-reqwest-client/src/models/boost_query.rs index e7fa454c4..5138deeff 100644 --- a/rust/lance-namespace-reqwest-client/src/models/boost_query.rs +++ b/rust/lance-namespace-reqwest-client/src/models/boost_query.rs @@ -11,22 +11,25 @@ use crate::models; use serde::{Deserialize, Serialize}; +/// BoostQuery : Boost query that scores documents matching positive query higher and negative query lower #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] pub struct BoostQuery { + #[serde(rename = "positive")] + pub positive: Box, #[serde(rename = "negative")] - pub negative: serde_json::Value, + pub negative: Box, + /// Boost factor for negative query (default: 0.5) #[serde(rename = "negative_boost", skip_serializing_if = "Option::is_none")] pub negative_boost: Option, - #[serde(rename = "positive")] - pub positive: serde_json::Value, } impl BoostQuery { - pub fn new(negative: serde_json::Value, positive: serde_json::Value) -> BoostQuery { + /// Boost query that scores documents matching positive query higher and negative query lower + pub fn new(positive: models::FtsQuery, negative: models::FtsQuery) -> BoostQuery { BoostQuery { - negative, + positive: Box::new(positive), + negative: Box::new(negative), negative_boost: None, - positive, } } } diff --git a/rust/lance-namespace-reqwest-client/src/models/fts_query.rs b/rust/lance-namespace-reqwest-client/src/models/fts_query.rs index efc93446d..e0e8f2003 100644 --- a/rust/lance-namespace-reqwest-client/src/models/fts_query.rs +++ b/rust/lance-namespace-reqwest-client/src/models/fts_query.rs @@ -11,19 +11,31 @@ use crate::models; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum FtsQuery { - FtsQueryOneOf(Box), - FtsQueryOneOf1(Box), - FtsQueryOneOf2(Box), - FtsQueryOneOf3(Box), - FtsQueryOneOf4(Box), +/// FtsQuery : Full-text search query. Exactly one query type field must be provided. This structure follows the same pattern as AlterTransactionAction to minimize differences and compatibility issues across codegen in different languages. +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +pub struct FtsQuery { + #[serde(rename = "match", skip_serializing_if = "Option::is_none")] + pub r#match: Option>, + #[serde(rename = "phrase", skip_serializing_if = "Option::is_none")] + pub phrase: Option>, + #[serde(rename = "boost", skip_serializing_if = "Option::is_none")] + pub boost: Option>, + #[serde(rename = "multi_match", skip_serializing_if = "Option::is_none")] + pub multi_match: Option>, + #[serde(rename = "boolean", skip_serializing_if = "Option::is_none")] + pub boolean: Option>, } -impl Default for FtsQuery { - fn default() -> Self { - Self::FtsQueryOneOf(Default::default()) +impl FtsQuery { + /// Full-text search query. Exactly one query type field must be provided. This structure follows the same pattern as AlterTransactionAction to minimize differences and compatibility issues across codegen in different languages. + pub fn new() -> FtsQuery { + FtsQuery { + r#match: None, + phrase: None, + boost: None, + multi_match: None, + boolean: None, + } } } diff --git a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of.rs b/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of.rs deleted file mode 100644 index 7aeedd326..000000000 --- a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of.rs +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Lance REST Namespace Specification - * - * This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use crate::models; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FtsQueryOneOf { - #[serde(rename = "match")] - pub r#match: Box, -} - -impl FtsQueryOneOf { - pub fn new(r#match: models::MatchQuery) -> FtsQueryOneOf { - FtsQueryOneOf { - r#match: Box::new(r#match), - } - } -} - diff --git a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_1.rs b/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_1.rs deleted file mode 100644 index 7d6a2bc18..000000000 --- a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_1.rs +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Lance REST Namespace Specification - * - * This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use crate::models; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FtsQueryOneOf1 { - #[serde(rename = "phrase")] - pub phrase: Box, -} - -impl FtsQueryOneOf1 { - pub fn new(phrase: models::PhraseQuery) -> FtsQueryOneOf1 { - FtsQueryOneOf1 { - phrase: Box::new(phrase), - } - } -} - diff --git a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_2.rs b/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_2.rs deleted file mode 100644 index 2c6374e2d..000000000 --- a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_2.rs +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Lance REST Namespace Specification - * - * This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use crate::models; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FtsQueryOneOf2 { - #[serde(rename = "boost")] - pub boost: Box, -} - -impl FtsQueryOneOf2 { - pub fn new(boost: models::BoostQuery) -> FtsQueryOneOf2 { - FtsQueryOneOf2 { - boost: Box::new(boost), - } - } -} - diff --git a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_4.rs b/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_4.rs deleted file mode 100644 index 167aa36b7..000000000 --- a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_4.rs +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Lance REST Namespace Specification - * - * This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. - * - * The version of the OpenAPI document: 0.0.1 - * - * Generated by: https://openapi-generator.tech - */ - -use crate::models; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FtsQueryOneOf4 { - #[serde(rename = "boolean")] - pub boolean: Box, -} - -impl FtsQueryOneOf4 { - pub fn new(boolean: models::BooleanQuery) -> FtsQueryOneOf4 { - FtsQueryOneOf4 { - boolean: Box::new(boolean), - } - } -} - diff --git a/rust/lance-namespace-reqwest-client/src/models/mod.rs b/rust/lance-namespace-reqwest-client/src/models/mod.rs index 31205faec..72051f662 100644 --- a/rust/lance-namespace-reqwest-client/src/models/mod.rs +++ b/rust/lance-namespace-reqwest-client/src/models/mod.rs @@ -62,16 +62,6 @@ pub mod error_response; pub use self::error_response::ErrorResponse; pub mod fts_query; pub use self::fts_query::FtsQuery; -pub mod fts_query_one_of; -pub use self::fts_query_one_of::FtsQueryOneOf; -pub mod fts_query_one_of_1; -pub use self::fts_query_one_of_1::FtsQueryOneOf1; -pub mod fts_query_one_of_2; -pub use self::fts_query_one_of_2::FtsQueryOneOf2; -pub mod fts_query_one_of_3; -pub use self::fts_query_one_of_3::FtsQueryOneOf3; -pub mod fts_query_one_of_4; -pub use self::fts_query_one_of_4::FtsQueryOneOf4; pub mod index_list_item_response; pub use self::index_list_item_response::IndexListItemResponse; pub mod index_list_request; @@ -116,6 +106,8 @@ pub mod phrase_query; pub use self::phrase_query::PhraseQuery; pub mod query_request; pub use self::query_request::QueryRequest; +pub mod query_request_full_text_query; +pub use self::query_request_full_text_query::QueryRequestFullTextQuery; pub mod register_table_request; pub use self::register_table_request::RegisterTableRequest; pub mod register_table_response; diff --git a/rust/lance-namespace-reqwest-client/src/models/query_request.rs b/rust/lance-namespace-reqwest-client/src/models/query_request.rs index 837c59c91..84a386d73 100644 --- a/rust/lance-namespace-reqwest-client/src/models/query_request.rs +++ b/rust/lance-namespace-reqwest-client/src/models/query_request.rs @@ -35,9 +35,8 @@ pub struct QueryRequest { /// Optional SQL filter expression #[serde(rename = "filter", default, with = "::serde_with::rust::double_option", skip_serializing_if = "Option::is_none")] pub filter: Option>, - /// Optional full-text search query (only string query supported) #[serde(rename = "full_text_query", skip_serializing_if = "Option::is_none")] - pub full_text_query: Option>, + pub full_text_query: Option>, /// Number of results to return #[serde(rename = "k")] pub k: i32, diff --git a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_3.rs b/rust/lance-namespace-reqwest-client/src/models/query_request_full_text_query.rs similarity index 55% rename from rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_3.rs rename to rust/lance-namespace-reqwest-client/src/models/query_request_full_text_query.rs index 86a8dc8f0..9b748027e 100644 --- a/rust/lance-namespace-reqwest-client/src/models/fts_query_one_of_3.rs +++ b/rust/lance-namespace-reqwest-client/src/models/query_request_full_text_query.rs @@ -11,16 +11,21 @@ use crate::models; use serde::{Deserialize, Serialize}; +/// QueryRequestFullTextQuery : Optional full-text search query. Provide either string_query or structured_query, not both. #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] -pub struct FtsQueryOneOf3 { - #[serde(rename = "multi_match")] - pub multi_match: Box, +pub struct QueryRequestFullTextQuery { + #[serde(rename = "string_query", skip_serializing_if = "Option::is_none")] + pub string_query: Option>, + #[serde(rename = "structured_query", skip_serializing_if = "Option::is_none")] + pub structured_query: Option>, } -impl FtsQueryOneOf3 { - pub fn new(multi_match: models::MultiMatchQuery) -> FtsQueryOneOf3 { - FtsQueryOneOf3 { - multi_match: Box::new(multi_match), +impl QueryRequestFullTextQuery { + /// Optional full-text search query. Provide either string_query or structured_query, not both. + pub fn new() -> QueryRequestFullTextQuery { + QueryRequestFullTextQuery { + string_query: None, + structured_query: None, } } } From 77b83d68252700d55fc14b8c06805aac5ae49d01 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sat, 19 Jul 2025 16:42:50 -0700 Subject: [PATCH 02/10] docs: update Java SDK documentation to reflect structured FTS support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Removed structured full-text search from known limitations - Added comprehensive examples for boolean, phrase, and boost queries - Updated all FTS examples to use new QueryRequestFullTextQuery wrapper - Added advanced structured FTS section with detailed code examples The Java SDK now fully supports complex full-text search queries thanks to the properties-based pattern implementation. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- docs/src/user-guide/java-sdk.md | 87 ++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 19 deletions(-) diff --git a/docs/src/user-guide/java-sdk.md b/docs/src/user-guide/java-sdk.md index 10d6f9306..c29b45162 100644 --- a/docs/src/user-guide/java-sdk.md +++ b/docs/src/user-guide/java-sdk.md @@ -310,19 +310,82 @@ waitForIndexComplete("my_table", "content_idx", 30); // Perform full-text search import com.lancedb.lance.namespace.model.StringFtsQuery; +import com.lancedb.lance.namespace.model.QueryRequestFullTextQuery; QueryRequest textQuery = new QueryRequest(); textQuery.setName("my_table"); textQuery.setK(10); +// Simple string query +QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); StringFtsQuery fts = new StringFtsQuery(); fts.setQuery("search terms"); // The search query fts.setColumns(Arrays.asList("content", "title")); // Optional: columns to search -textQuery.setFullTextQuery(fts); +fullTextQuery.setStringQuery(fts); +textQuery.setFullTextQuery(fullTextQuery); byte[] results = namespace.queryTable(textQuery); ``` +##### Advanced: Structured Full-Text Search + +The Java SDK now supports complex structured full-text queries including boolean queries, phrase queries, and boosted queries: + +```java +import com.lancedb.lance.namespace.model.*; + +// Boolean query example: (must contain "important" AND should contain "feature" OR "update") +QueryRequest structuredQuery = new QueryRequest(); +structuredQuery.setName("my_table"); +structuredQuery.setK(10); + +QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); +StructuredFtsQuery structured = new StructuredFtsQuery(); +FtsQuery ftsQuery = new FtsQuery(); + +// Create a boolean query +BooleanQuery boolQuery = new BooleanQuery(); + +// Must clause: documents must contain "important" +FtsQuery mustQuery = new FtsQuery(); +MatchQuery mustMatch = new MatchQuery(); +mustMatch.setTerms("important"); +mustMatch.setColumn("content"); +mustQuery.setMatch(mustMatch); +boolQuery.setMust(Arrays.asList(mustQuery)); + +// Should clauses: documents should contain "feature" OR "update" +FtsQuery shouldQuery1 = new FtsQuery(); +MatchQuery shouldMatch1 = new MatchQuery(); +shouldMatch1.setTerms("feature"); +shouldQuery1.setMatch(shouldMatch1); + +FtsQuery shouldQuery2 = new FtsQuery(); +MatchQuery shouldMatch2 = new MatchQuery(); +shouldMatch2.setTerms("update"); +shouldQuery2.setMatch(shouldMatch2); +boolQuery.setShould(Arrays.asList(shouldQuery1, shouldQuery2)); + +// Must NOT clause: exclude documents with "deprecated" +FtsQuery mustNotQuery = new FtsQuery(); +MatchQuery mustNotMatch = new MatchQuery(); +mustNotMatch.setTerms("deprecated"); +mustNotQuery.setMatch(mustNotMatch); +boolQuery.setMustNot(Arrays.asList(mustNotQuery)); + +ftsQuery.setBoolean(boolQuery); +structured.setQuery(ftsQuery); +fullTextQuery.setStructuredQuery(structured); +structuredQuery.setFullTextQuery(fullTextQuery); + +byte[] boolResults = namespace.queryTable(structuredQuery); +``` + +Other supported query types include: +- **Phrase Query**: Find exact phrases with optional slop (word distance) +- **Boost Query**: Boost documents matching certain criteria +- **Multi-Match Query**: Search across multiple fields with different weights + #### Hybrid Search Combining vector similarity search with full-text search often provides more relevant results than using either method alone. This is especially useful for semantic search applications where both conceptual similarity and keyword matching are important. @@ -338,10 +401,12 @@ hybridQuery.setVector(queryVector); hybridQuery.setK(10); // Text search component +QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); StringFtsQuery fts = new StringFtsQuery(); fts.setQuery("search terms"); fts.setColumns(Arrays.asList("content", "title")); // Optional: columns to search -hybridQuery.setFullTextQuery(fts); +fullTextQuery.setStringQuery(fts); +hybridQuery.setFullTextQuery(fullTextQuery); // Optional: Add filters hybridQuery.setFilter("date > '2024-01-01'"); @@ -567,11 +632,6 @@ Due to limitations in the OpenAPI code generator for Java, some advanced feature - **Workaround**: Use `Arrays.asList("col1", "col2")` to specify columns - **Not Supported**: Advanced column specifications with include/exclude patterns -### 3. Structured Full-Text Search -- **Limitation**: Only simple string queries are supported via `StringFtsQuery` -- **Workaround**: Use basic text search with `query.setFullTextQuery(stringQuery)` -- **Not Supported**: Complex boolean queries, phrase queries, or boosted queries - ### Example of Supported vs Unsupported Features ```java @@ -593,20 +653,9 @@ query.setColumns(Arrays.asList("id", "name", "score")); // ColumnsObject cols = new ColumnsObject(); // cols.setInclude(Arrays.asList("*")); // cols.setExclude(Arrays.asList("embedding")); - -// ✅ SUPPORTED: Simple text search -StringFtsQuery fts = new StringFtsQuery(); -fts.setQuery("search terms"); -fts.setColumns(Arrays.asList("title", "content")); -query.setFullTextQuery(fts); - -// ❌ NOT SUPPORTED: Structured boolean queries -// BooleanQuery bool = new BooleanQuery(); -// bool.setMust(Arrays.asList(matchQuery1)); -// bool.setShould(Arrays.asList(matchQuery2)); ``` -These limitations are due to the OpenAPI generator's inability to properly handle `oneOf` types in the specification. The simplified types ensure the SDK works reliably for the most common use cases. +These limitations are due to the OpenAPI generator's inability to properly handle certain `oneOf` types in the specification. The simplified types ensure the SDK works reliably for the most common use cases. ## Additional Resources From e144b152dd5d193599a413faa9a004771ee8edda Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sat, 19 Jul 2025 16:55:03 -0700 Subject: [PATCH 03/10] improve doc --- docs/src/spec/rest.yaml | 23 +- docs/src/user-guide/java-sdk.md | 42 +--- java/lance-namespace-apache-client/README.md | 1 + .../api/openapi.yaml | 44 +++- .../docs/QueryRequest.md | 2 +- .../docs/QueryRequestVector.md | 15 ++ .../lance/namespace/model/QueryRequest.java | 36 +-- .../namespace/model/QueryRequestVector.java | 226 ++++++++++++++++++ .../server/springboot/model/QueryRequest.java | 26 +- .../springboot/model/QueryRequestVector.java | 146 +++++++++++ .../lance_namespace_urllib3_client/README.md | 1 + .../docs/QueryRequest.md | 2 +- .../docs/QueryRequestVector.md | 31 +++ .../__init__.py | 1 + .../models/__init__.py | 1 + .../models/query_request.py | 8 +- .../models/query_request_vector.py | 89 +++++++ .../test/test_query_request.py | 24 +- .../test/test_query_request_vector.py | 58 +++++ rust/lance-namespace-reqwest-client/README.md | 1 + .../docs/QueryRequest.md | 2 +- .../docs/QueryRequestVector.md | 12 + .../src/models/mod.rs | 2 + .../src/models/query_request.rs | 7 +- .../src/models/query_request_vector.rs | 34 +++ 25 files changed, 721 insertions(+), 113 deletions(-) create mode 100644 java/lance-namespace-apache-client/docs/QueryRequestVector.md create mode 100644 java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequestVector.java create mode 100644 java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequestVector.java create mode 100644 python/lance_namespace_urllib3_client/docs/QueryRequestVector.md create mode 100644 python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request_vector.py create mode 100644 python/lance_namespace_urllib3_client/test/test_query_request_vector.py create mode 100644 rust/lance-namespace-reqwest-client/docs/QueryRequestVector.md create mode 100644 rust/lance-namespace-reqwest-client/src/models/query_request_vector.rs diff --git a/docs/src/spec/rest.yaml b/docs/src/spec/rest.yaml index 21c8cbb69..bcc4aad84 100644 --- a/docs/src/spec/rest.yaml +++ b/docs/src/spec/rest.yaml @@ -1577,11 +1577,24 @@ components: format: float description: Upper bound for search vector: - type: array - items: - type: number - format: float - description: Query vector for similarity search (single vector only) + type: object + nullable: true + description: Query vector(s) for similarity search. Provide either single_vector or multi_vector, not both. + properties: + single_vector: + type: array + items: + type: number + format: float + description: Single query vector + multi_vector: + type: array + items: + type: array + items: + type: number + format: float + description: Multiple query vectors for batch search vector_column: type: - string diff --git a/docs/src/user-guide/java-sdk.md b/docs/src/user-guide/java-sdk.md index c29b45162..f028fd1c0 100644 --- a/docs/src/user-guide/java-sdk.md +++ b/docs/src/user-guide/java-sdk.md @@ -184,6 +184,7 @@ Query results are returned in Arrow File format. Use `ArrowFileReader` to read t ```java import com.lancedb.lance.namespace.model.QueryRequest; +import com.lancedb.lance.namespace.model.QueryRequestVector; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.ArrowBlock; import org.apache.arrow.vector.ipc.SeekableReadChannel; @@ -196,11 +197,13 @@ import java.util.Arrays; QueryRequest queryRequest = new QueryRequest(); queryRequest.setName("my_table"); -// Set query vector (128-dimensional) +// Single vector search (most common use case) List queryVector = new ArrayList<>(); for (int i = 0; i < 128; i++) { queryVector.add((float) Math.random()); } + +// For single vector, you can set it directly queryRequest.setVector(queryVector); queryRequest.setK(5); // Get top 5 results @@ -619,43 +622,6 @@ System.out.println("Updated rows: " + response.getNumUpdatedRows()); System.out.println("Inserted rows: " + response.getNumInsertedRows()); ``` -## Known Limitations - -Due to limitations in the OpenAPI code generator for Java, some advanced features that use `oneOf` polymorphic types are not fully supported in this SDK. The SDK has been simplified to support the most common use cases: - -### 1. Multi-Vector Queries -- **Limitation**: Only single vector queries are supported -- **Workaround**: Submit one vector at a time rather than batching multiple vectors - -### 2. Complex Column Specifications -- **Limitation**: Only simple column lists (array of strings) are supported -- **Workaround**: Use `Arrays.asList("col1", "col2")` to specify columns -- **Not Supported**: Advanced column specifications with include/exclude patterns - -### Example of Supported vs Unsupported Features - -```java -// ✅ SUPPORTED: Simple vector query -QueryRequest query = new QueryRequest(); -query.setName("my_table"); -query.setK(10); -List vector = Arrays.asList(0.1f, 0.2f, 0.3f, ...); -query.setVector(vector); - -// ❌ NOT SUPPORTED: Multi-vector query -// List> vectors = Arrays.asList(vector1, vector2, vector3); -// query.setVector(vectors); // This won't compile - -// ✅ SUPPORTED: Simple column selection -query.setColumns(Arrays.asList("id", "name", "score")); - -// ❌ NOT SUPPORTED: Complex column specification -// ColumnsObject cols = new ColumnsObject(); -// cols.setInclude(Arrays.asList("*")); -// cols.setExclude(Arrays.asList("embedding")); -``` - -These limitations are due to the OpenAPI generator's inability to properly handle certain `oneOf` types in the specification. The simplified types ensure the SDK works reliably for the most common use cases. ## Additional Resources diff --git a/java/lance-namespace-apache-client/README.md b/java/lance-namespace-apache-client/README.md index faafaf33c..16ec7f09b 100644 --- a/java/lance-namespace-apache-client/README.md +++ b/java/lance-namespace-apache-client/README.md @@ -197,6 +197,7 @@ Class | Method | HTTP request | Description - [PhraseQuery](docs/PhraseQuery.md) - [QueryRequest](docs/QueryRequest.md) - [QueryRequestFullTextQuery](docs/QueryRequestFullTextQuery.md) + - [QueryRequestVector](docs/QueryRequestVector.md) - [RegisterTableRequest](docs/RegisterTableRequest.md) - [RegisterTableResponse](docs/RegisterTableResponse.md) - [SetPropertyMode](docs/SetPropertyMode.md) diff --git a/java/lance-namespace-apache-client/api/openapi.yaml b/java/lance-namespace-apache-client/api/openapi.yaml index e8cd12ce8..fcdddde97 100644 --- a/java/lance-namespace-apache-client/api/openapi.yaml +++ b/java/lance-namespace-apache-client/api/openapi.yaml @@ -2087,8 +2087,14 @@ components: - namespace - namespace vector: - - 1.0246457 - - 1.0246457 + single_vector: + - 1.0246457 + - 1.0246457 + multi_vector: + - - 1.4894159 + - 1.4894159 + - - 1.4894159 + - 1.4894159 properties: name: type: string @@ -2160,11 +2166,7 @@ components: nullable: true type: number vector: - description: Query vector for similarity search (single vector only) - items: - format: float - type: number - type: array + $ref: '#/components/schemas/QueryRequest_vector' vector_column: description: Name of the vector column to search nullable: true @@ -3128,4 +3130,32 @@ components: structured_query: $ref: '#/components/schemas/StructuredFtsQuery' nullable: true + QueryRequest_vector: + description: "Query vector(s) for similarity search. Provide either single_vector\ + \ or multi_vector, not both." + example: + single_vector: + - 1.0246457 + - 1.0246457 + multi_vector: + - - 1.4894159 + - 1.4894159 + - - 1.4894159 + - 1.4894159 + properties: + single_vector: + description: Single query vector + items: + format: float + type: number + type: array + multi_vector: + description: Multiple query vectors for batch search + items: + items: + format: float + type: number + type: array + type: array + nullable: true diff --git a/java/lance-namespace-apache-client/docs/QueryRequest.md b/java/lance-namespace-apache-client/docs/QueryRequest.md index d98905487..81ba8e82a 100644 --- a/java/lance-namespace-apache-client/docs/QueryRequest.md +++ b/java/lance-namespace-apache-client/docs/QueryRequest.md @@ -23,7 +23,7 @@ |**prefilter** | **Boolean** | Whether to apply filtering before vector search | [optional] | |**refineFactor** | **Integer** | Refine factor for search | [optional] | |**upperBound** | **Float** | Upper bound for search | [optional] | -|**vector** | **List<Float>** | Query vector for similarity search (single vector only) | | +|**vector** | [**QueryRequestVector**](QueryRequestVector.md) | | | |**vectorColumn** | **String** | Name of the vector column to search | [optional] | |**version** | **Long** | Table version to query | [optional] | |**withRowId** | **Boolean** | If true, return the row id as a column called `_rowid` | [optional] | diff --git a/java/lance-namespace-apache-client/docs/QueryRequestVector.md b/java/lance-namespace-apache-client/docs/QueryRequestVector.md new file mode 100644 index 000000000..a2606dd5a --- /dev/null +++ b/java/lance-namespace-apache-client/docs/QueryRequestVector.md @@ -0,0 +1,15 @@ + + +# QueryRequestVector + +Query vector(s) for similarity search. Provide either single_vector or multi_vector, not both. + +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +|**singleVector** | **List<Float>** | Single query vector | [optional] | +|**multiVector** | **List<List<Float>>** | Multiple query vectors for batch search | [optional] | + + + diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequest.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequest.java index 3937b045d..5e5f06d3a 100644 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequest.java +++ b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequest.java @@ -121,7 +121,7 @@ public class QueryRequest { private JsonNullable upperBound = JsonNullable.undefined(); public static final String JSON_PROPERTY_VECTOR = "vector"; - @javax.annotation.Nonnull private List vector = new ArrayList<>(); + @javax.annotation.Nonnull private QueryRequestVector vector; public static final String JSON_PROPERTY_VECTOR_COLUMN = "vector_column"; @@ -627,35 +627,27 @@ public void setUpperBound(@javax.annotation.Nullable Float upperBound) { this.upperBound = JsonNullable.of(upperBound); } - public QueryRequest vector(@javax.annotation.Nonnull List vector) { + public QueryRequest vector(@javax.annotation.Nonnull QueryRequestVector vector) { this.vector = vector; return this; } - public QueryRequest addVectorItem(Float vectorItem) { - if (this.vector == null) { - this.vector = new ArrayList<>(); - } - this.vector.add(vectorItem); - return this; - } - /** - * Query vector for similarity search (single vector only) + * Get vector * * @return vector */ @javax.annotation.Nonnull @JsonProperty(JSON_PROPERTY_VECTOR) @JsonInclude(value = JsonInclude.Include.ALWAYS) - public List getVector() { + public QueryRequestVector getVector() { return vector; } @JsonProperty(JSON_PROPERTY_VECTOR) @JsonInclude(value = JsonInclude.Include.ALWAYS) - public void setVector(@javax.annotation.Nonnull List vector) { + public void setVector(@javax.annotation.Nonnull QueryRequestVector vector) { this.vector = vector; } @@ -1148,23 +1140,7 @@ public String toUrlQueryString(String prefix) { // add `vector` to the URL query string if (getVector() != null) { - for (int i = 0; i < getVector().size(); i++) { - try { - joiner.add( - String.format( - "%svector%s%s=%s", - prefix, - suffix, - "".equals(suffix) - ? "" - : String.format("%s%d%s", containerPrefix, i, containerSuffix), - URLEncoder.encode(String.valueOf(getVector().get(i)), "UTF-8") - .replaceAll("\\+", "%20"))); - } catch (UnsupportedEncodingException e) { - // Should never happen, UTF-8 is always supported - throw new RuntimeException(e); - } - } + joiner.add(getVector().toUrlQueryString(prefix + "vector" + suffix)); } // add `vector_column` to the URL query string diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequestVector.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequestVector.java new file mode 100644 index 000000000..4cf7f31f6 --- /dev/null +++ b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/QueryRequestVector.java @@ -0,0 +1,226 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonTypeName; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.StringJoiner; + +/** + * Query vector(s) for similarity search. Provide either single_vector or multi_vector, not both. + */ +@JsonPropertyOrder({ + QueryRequestVector.JSON_PROPERTY_SINGLE_VECTOR, + QueryRequestVector.JSON_PROPERTY_MULTI_VECTOR +}) +@JsonTypeName("QueryRequest_vector") +@javax.annotation.Generated( + value = "org.openapitools.codegen.languages.JavaClientCodegen", + comments = "Generator version: 7.12.0") +public class QueryRequestVector { + public static final String JSON_PROPERTY_SINGLE_VECTOR = "single_vector"; + @javax.annotation.Nullable private List singleVector = new ArrayList<>(); + + public static final String JSON_PROPERTY_MULTI_VECTOR = "multi_vector"; + @javax.annotation.Nullable private List> multiVector = new ArrayList<>(); + + public QueryRequestVector() {} + + public QueryRequestVector singleVector(@javax.annotation.Nullable List singleVector) { + + this.singleVector = singleVector; + return this; + } + + public QueryRequestVector addSingleVectorItem(Float singleVectorItem) { + if (this.singleVector == null) { + this.singleVector = new ArrayList<>(); + } + this.singleVector.add(singleVectorItem); + return this; + } + + /** + * Single query vector + * + * @return singleVector + */ + @javax.annotation.Nullable + @JsonProperty(JSON_PROPERTY_SINGLE_VECTOR) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public List getSingleVector() { + return singleVector; + } + + @JsonProperty(JSON_PROPERTY_SINGLE_VECTOR) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setSingleVector(@javax.annotation.Nullable List singleVector) { + this.singleVector = singleVector; + } + + public QueryRequestVector multiVector(@javax.annotation.Nullable List> multiVector) { + + this.multiVector = multiVector; + return this; + } + + public QueryRequestVector addMultiVectorItem(List multiVectorItem) { + if (this.multiVector == null) { + this.multiVector = new ArrayList<>(); + } + this.multiVector.add(multiVectorItem); + return this; + } + + /** + * Multiple query vectors for batch search + * + * @return multiVector + */ + @javax.annotation.Nullable + @JsonProperty(JSON_PROPERTY_MULTI_VECTOR) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public List> getMultiVector() { + return multiVector; + } + + @JsonProperty(JSON_PROPERTY_MULTI_VECTOR) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setMultiVector(@javax.annotation.Nullable List> multiVector) { + this.multiVector = multiVector; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + QueryRequestVector queryRequestVector = (QueryRequestVector) o; + return Objects.equals(this.singleVector, queryRequestVector.singleVector) + && Objects.equals(this.multiVector, queryRequestVector.multiVector); + } + + @Override + public int hashCode() { + return Objects.hash(singleVector, multiVector); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class QueryRequestVector {\n"); + sb.append(" singleVector: ").append(toIndentedString(singleVector)).append("\n"); + sb.append(" multiVector: ").append(toIndentedString(multiVector)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + + /** + * Convert the instance into URL query string. + * + * @return URL query string + */ + public String toUrlQueryString() { + return toUrlQueryString(null); + } + + /** + * Convert the instance into URL query string. + * + * @param prefix prefix of the query string + * @return URL query string + */ + public String toUrlQueryString(String prefix) { + String suffix = ""; + String containerSuffix = ""; + String containerPrefix = ""; + if (prefix == null) { + // style=form, explode=true, e.g. /pet?name=cat&type=manx + prefix = ""; + } else { + // deepObject style e.g. /pet?id[name]=cat&id[type]=manx + prefix = prefix + "["; + suffix = "]"; + containerSuffix = "]"; + containerPrefix = "["; + } + + StringJoiner joiner = new StringJoiner("&"); + + // add `single_vector` to the URL query string + if (getSingleVector() != null) { + for (int i = 0; i < getSingleVector().size(); i++) { + try { + joiner.add( + String.format( + "%ssingle_vector%s%s=%s", + prefix, + suffix, + "".equals(suffix) + ? "" + : String.format("%s%d%s", containerPrefix, i, containerSuffix), + URLEncoder.encode(String.valueOf(getSingleVector().get(i)), "UTF-8") + .replaceAll("\\+", "%20"))); + } catch (UnsupportedEncodingException e) { + // Should never happen, UTF-8 is always supported + throw new RuntimeException(e); + } + } + } + + // add `multi_vector` to the URL query string + if (getMultiVector() != null) { + for (int i = 0; i < getMultiVector().size(); i++) { + try { + joiner.add( + String.format( + "%smulti_vector%s%s=%s", + prefix, + suffix, + "".equals(suffix) + ? "" + : String.format("%s%d%s", containerPrefix, i, containerSuffix), + URLEncoder.encode(String.valueOf(getMultiVector().get(i)), "UTF-8") + .replaceAll("\\+", "%20"))); + } catch (UnsupportedEncodingException e) { + // Should never happen, UTF-8 is always supported + throw new RuntimeException(e); + } + } + } + + return joiner.toString(); + } +} diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequest.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequest.java index c033d6854..204cab247 100644 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequest.java +++ b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequest.java @@ -63,7 +63,7 @@ public class QueryRequest { private Float upperBound = null; - @Valid private List vector = new ArrayList<>(); + private QueryRequestVector vector; private String vectorColumn = null; @@ -76,7 +76,7 @@ public QueryRequest() { } /** Constructor with only required parameters */ - public QueryRequest(String name, List namespace, Integer k, List vector) { + public QueryRequest(String name, List namespace, Integer k, QueryRequestVector vector) { this.name = name; this.namespace = namespace; this.k = k; @@ -467,35 +467,25 @@ public void setUpperBound(Float upperBound) { this.upperBound = upperBound; } - public QueryRequest vector(List vector) { + public QueryRequest vector(QueryRequestVector vector) { this.vector = vector; return this; } - public QueryRequest addVectorItem(Float vectorItem) { - if (this.vector == null) { - this.vector = new ArrayList<>(); - } - this.vector.add(vectorItem); - return this; - } - /** - * Query vector for similarity search (single vector only) + * Get vector * * @return vector */ @NotNull - @Schema( - name = "vector", - description = "Query vector for similarity search (single vector only)", - requiredMode = Schema.RequiredMode.REQUIRED) + @Valid + @Schema(name = "vector", requiredMode = Schema.RequiredMode.REQUIRED) @JsonProperty("vector") - public List getVector() { + public QueryRequestVector getVector() { return vector; } - public void setVector(List vector) { + public void setVector(QueryRequestVector vector) { this.vector = vector; } diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequestVector.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequestVector.java new file mode 100644 index 000000000..555066ea9 --- /dev/null +++ b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/QueryRequestVector.java @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.server.springboot.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.swagger.v3.oas.annotations.media.Schema; + +import javax.annotation.Generated; +import javax.validation.Valid; +import javax.validation.constraints.*; + +import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Query vector(s) for similarity search. Provide either single_vector or multi_vector, not both. + */ +@Schema( + name = "QueryRequest_vector", + description = + "Query vector(s) for similarity search. Provide either single_vector or multi_vector, not both.") +@JsonTypeName("QueryRequest_vector") +@Generated( + value = "org.openapitools.codegen.languages.SpringCodegen", + comments = "Generator version: 7.12.0") +public class QueryRequestVector { + + @Valid private List singleVector = new ArrayList<>(); + + @Valid private List> multiVector = new ArrayList<>(); + + public QueryRequestVector singleVector(List singleVector) { + this.singleVector = singleVector; + return this; + } + + public QueryRequestVector addSingleVectorItem(Float singleVectorItem) { + if (this.singleVector == null) { + this.singleVector = new ArrayList<>(); + } + this.singleVector.add(singleVectorItem); + return this; + } + + /** + * Single query vector + * + * @return singleVector + */ + @Schema( + name = "single_vector", + description = "Single query vector", + requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("single_vector") + public List getSingleVector() { + return singleVector; + } + + public void setSingleVector(List singleVector) { + this.singleVector = singleVector; + } + + public QueryRequestVector multiVector(List> multiVector) { + this.multiVector = multiVector; + return this; + } + + public QueryRequestVector addMultiVectorItem(List multiVectorItem) { + if (this.multiVector == null) { + this.multiVector = new ArrayList<>(); + } + this.multiVector.add(multiVectorItem); + return this; + } + + /** + * Multiple query vectors for batch search + * + * @return multiVector + */ + @Valid + @Schema( + name = "multi_vector", + description = "Multiple query vectors for batch search", + requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("multi_vector") + public List> getMultiVector() { + return multiVector; + } + + public void setMultiVector(List> multiVector) { + this.multiVector = multiVector; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + QueryRequestVector queryRequestVector = (QueryRequestVector) o; + return Objects.equals(this.singleVector, queryRequestVector.singleVector) + && Objects.equals(this.multiVector, queryRequestVector.multiVector); + } + + @Override + public int hashCode() { + return Objects.hash(singleVector, multiVector); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class QueryRequestVector {\n"); + sb.append(" singleVector: ").append(toIndentedString(singleVector)).append("\n"); + sb.append(" multiVector: ").append(toIndentedString(multiVector)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} diff --git a/python/lance_namespace_urllib3_client/README.md b/python/lance_namespace_urllib3_client/README.md index 74987d059..033e36d86 100644 --- a/python/lance_namespace_urllib3_client/README.md +++ b/python/lance_namespace_urllib3_client/README.md @@ -174,6 +174,7 @@ Class | Method | HTTP request | Description - [PhraseQuery](docs/PhraseQuery.md) - [QueryRequest](docs/QueryRequest.md) - [QueryRequestFullTextQuery](docs/QueryRequestFullTextQuery.md) + - [QueryRequestVector](docs/QueryRequestVector.md) - [RegisterTableRequest](docs/RegisterTableRequest.md) - [RegisterTableResponse](docs/RegisterTableResponse.md) - [SetPropertyMode](docs/SetPropertyMode.md) diff --git a/python/lance_namespace_urllib3_client/docs/QueryRequest.md b/python/lance_namespace_urllib3_client/docs/QueryRequest.md index 28706a3a7..17c794909 100644 --- a/python/lance_namespace_urllib3_client/docs/QueryRequest.md +++ b/python/lance_namespace_urllib3_client/docs/QueryRequest.md @@ -21,7 +21,7 @@ Name | Type | Description | Notes **prefilter** | **bool** | Whether to apply filtering before vector search | [optional] **refine_factor** | **int** | Refine factor for search | [optional] **upper_bound** | **float** | Upper bound for search | [optional] -**vector** | **List[float]** | Query vector for similarity search (single vector only) | +**vector** | [**QueryRequestVector**](QueryRequestVector.md) | | **vector_column** | **str** | Name of the vector column to search | [optional] **version** | **int** | Table version to query | [optional] **with_row_id** | **bool** | If true, return the row id as a column called `_rowid` | [optional] diff --git a/python/lance_namespace_urllib3_client/docs/QueryRequestVector.md b/python/lance_namespace_urllib3_client/docs/QueryRequestVector.md new file mode 100644 index 000000000..14b26f738 --- /dev/null +++ b/python/lance_namespace_urllib3_client/docs/QueryRequestVector.md @@ -0,0 +1,31 @@ +# QueryRequestVector + +Query vector(s) for similarity search. Provide either single_vector or multi_vector, not both. + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**single_vector** | **List[float]** | Single query vector | [optional] +**multi_vector** | **List[List[float]]** | Multiple query vectors for batch search | [optional] + +## Example + +```python +from lance_namespace_urllib3_client.models.query_request_vector import QueryRequestVector + +# TODO update the JSON string below +json = "{}" +# create an instance of QueryRequestVector from a JSON string +query_request_vector_instance = QueryRequestVector.from_json(json) +# print the JSON string representation of the object +print(QueryRequestVector.to_json()) + +# convert the object into a dict +query_request_vector_dict = query_request_vector_instance.to_dict() +# create an instance of QueryRequestVector from a dict +query_request_vector_from_dict = QueryRequestVector.from_dict(query_request_vector_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/__init__.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/__init__.py index 5ea95d36c..c2d1991af 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/__init__.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/__init__.py @@ -88,6 +88,7 @@ from lance_namespace_urllib3_client.models.phrase_query import PhraseQuery from lance_namespace_urllib3_client.models.query_request import QueryRequest from lance_namespace_urllib3_client.models.query_request_full_text_query import QueryRequestFullTextQuery +from lance_namespace_urllib3_client.models.query_request_vector import QueryRequestVector from lance_namespace_urllib3_client.models.register_table_request import RegisterTableRequest from lance_namespace_urllib3_client.models.register_table_response import RegisterTableResponse from lance_namespace_urllib3_client.models.set_property_mode import SetPropertyMode diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/__init__.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/__init__.py index b48f10ba9..c6ebf7d88 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/__init__.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/__init__.py @@ -69,6 +69,7 @@ from lance_namespace_urllib3_client.models.phrase_query import PhraseQuery from lance_namespace_urllib3_client.models.query_request import QueryRequest from lance_namespace_urllib3_client.models.query_request_full_text_query import QueryRequestFullTextQuery +from lance_namespace_urllib3_client.models.query_request_vector import QueryRequestVector from lance_namespace_urllib3_client.models.register_table_request import RegisterTableRequest from lance_namespace_urllib3_client.models.register_table_response import RegisterTableResponse from lance_namespace_urllib3_client.models.set_property_mode import SetPropertyMode diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request.py index 1e9347f7c..5f3ed45ed 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request.py @@ -21,6 +21,7 @@ from typing import Any, ClassVar, Dict, List, Optional, Union from typing_extensions import Annotated from lance_namespace_urllib3_client.models.query_request_full_text_query import QueryRequestFullTextQuery +from lance_namespace_urllib3_client.models.query_request_vector import QueryRequestVector from typing import Optional, Set from typing_extensions import Self @@ -44,7 +45,7 @@ class QueryRequest(BaseModel): prefilter: Optional[StrictBool] = Field(default=None, description="Whether to apply filtering before vector search") refine_factor: Optional[Annotated[int, Field(strict=True, ge=0)]] = Field(default=None, description="Refine factor for search") upper_bound: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="Upper bound for search") - vector: List[Union[StrictFloat, StrictInt]] = Field(description="Query vector for similarity search (single vector only)") + vector: QueryRequestVector vector_column: Optional[StrictStr] = Field(default=None, description="Name of the vector column to search") version: Optional[Annotated[int, Field(strict=True, ge=0)]] = Field(default=None, description="Table version to query") with_row_id: Optional[StrictBool] = Field(default=None, description="If true, return the row id as a column called `_rowid`") @@ -92,6 +93,9 @@ def to_dict(self) -> Dict[str, Any]: # override the default output from pydantic by calling `to_dict()` of full_text_query if self.full_text_query: _dict['full_text_query'] = self.full_text_query.to_dict() + # override the default output from pydantic by calling `to_dict()` of vector + if self.vector: + _dict['vector'] = self.vector.to_dict() # set to None if bypass_vector_index (nullable) is None # and model_fields_set contains the field if self.bypass_vector_index is None and "bypass_vector_index" in self.model_fields_set: @@ -190,7 +194,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "prefilter": obj.get("prefilter"), "refine_factor": obj.get("refine_factor"), "upper_bound": obj.get("upper_bound"), - "vector": obj.get("vector"), + "vector": QueryRequestVector.from_dict(obj["vector"]) if obj.get("vector") is not None else None, "vector_column": obj.get("vector_column"), "version": obj.get("version"), "with_row_id": obj.get("with_row_id") diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request_vector.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request_vector.py new file mode 100644 index 000000000..35b311f25 --- /dev/null +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/query_request_vector.py @@ -0,0 +1,89 @@ +# coding: utf-8 + +""" + Lance REST Namespace Specification + + This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. + + The version of the OpenAPI document: 0.0.1 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt +from typing import Any, ClassVar, Dict, List, Optional, Union +from typing import Optional, Set +from typing_extensions import Self + +class QueryRequestVector(BaseModel): + """ + Query vector(s) for similarity search. Provide either single_vector or multi_vector, not both. + """ # noqa: E501 + single_vector: Optional[List[Union[StrictFloat, StrictInt]]] = Field(default=None, description="Single query vector") + multi_vector: Optional[List[List[Union[StrictFloat, StrictInt]]]] = Field(default=None, description="Multiple query vectors for batch search") + __properties: ClassVar[List[str]] = ["single_vector", "multi_vector"] + + model_config = ConfigDict( + populate_by_name=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of QueryRequestVector from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of QueryRequestVector from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "single_vector": obj.get("single_vector"), + "multi_vector": obj.get("multi_vector") + }) + return _obj + + diff --git a/python/lance_namespace_urllib3_client/test/test_query_request.py b/python/lance_namespace_urllib3_client/test/test_query_request.py index bfcf5ea07..ffb0a805b 100644 --- a/python/lance_namespace_urllib3_client/test/test_query_request.py +++ b/python/lance_namespace_urllib3_client/test/test_query_request.py @@ -111,9 +111,15 @@ def make_instance(self, include_optional) -> QueryRequest: prefilter = True, refine_factor = 0, upper_bound = 1.337, - vector = [ - 1.337 - ], + vector = lance_namespace_urllib3_client.models.query_request_vector.QueryRequest_vector( + single_vector = [ + 1.337 + ], + multi_vector = [ + [ + 1.337 + ] + ], ), vector_column = '', version = 0, with_row_id = True @@ -125,9 +131,15 @@ def make_instance(self, include_optional) -> QueryRequest: '' ], k = 0, - vector = [ - 1.337 - ], + vector = lance_namespace_urllib3_client.models.query_request_vector.QueryRequest_vector( + single_vector = [ + 1.337 + ], + multi_vector = [ + [ + 1.337 + ] + ], ), ) """ diff --git a/python/lance_namespace_urllib3_client/test/test_query_request_vector.py b/python/lance_namespace_urllib3_client/test/test_query_request_vector.py new file mode 100644 index 000000000..8924bb5a7 --- /dev/null +++ b/python/lance_namespace_urllib3_client/test/test_query_request_vector.py @@ -0,0 +1,58 @@ +# coding: utf-8 + +""" + Lance REST Namespace Specification + + This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. + + The version of the OpenAPI document: 0.0.1 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from lance_namespace_urllib3_client.models.query_request_vector import QueryRequestVector + +class TestQueryRequestVector(unittest.TestCase): + """QueryRequestVector unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> QueryRequestVector: + """Test QueryRequestVector + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `QueryRequestVector` + """ + model = QueryRequestVector() + if include_optional: + return QueryRequestVector( + single_vector = [ + 1.337 + ], + multi_vector = [ + [ + 1.337 + ] + ] + ) + else: + return QueryRequestVector( + ) + """ + + def testQueryRequestVector(self): + """Test QueryRequestVector""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/rust/lance-namespace-reqwest-client/README.md b/rust/lance-namespace-reqwest-client/README.md index 760af1051..c5571113b 100644 --- a/rust/lance-namespace-reqwest-client/README.md +++ b/rust/lance-namespace-reqwest-client/README.md @@ -113,6 +113,7 @@ Class | Method | HTTP request | Description - [PhraseQuery](docs/PhraseQuery.md) - [QueryRequest](docs/QueryRequest.md) - [QueryRequestFullTextQuery](docs/QueryRequestFullTextQuery.md) + - [QueryRequestVector](docs/QueryRequestVector.md) - [RegisterTableRequest](docs/RegisterTableRequest.md) - [RegisterTableResponse](docs/RegisterTableResponse.md) - [SetPropertyMode](docs/SetPropertyMode.md) diff --git a/rust/lance-namespace-reqwest-client/docs/QueryRequest.md b/rust/lance-namespace-reqwest-client/docs/QueryRequest.md index d7160cd94..0d174d468 100644 --- a/rust/lance-namespace-reqwest-client/docs/QueryRequest.md +++ b/rust/lance-namespace-reqwest-client/docs/QueryRequest.md @@ -20,7 +20,7 @@ Name | Type | Description | Notes **prefilter** | Option<**bool**> | Whether to apply filtering before vector search | [optional] **refine_factor** | Option<**i32**> | Refine factor for search | [optional] **upper_bound** | Option<**f32**> | Upper bound for search | [optional] -**vector** | **Vec** | Query vector for similarity search (single vector only) | +**vector** | [**models::QueryRequestVector**](QueryRequest_vector.md) | | **vector_column** | Option<**String**> | Name of the vector column to search | [optional] **version** | Option<**i64**> | Table version to query | [optional] **with_row_id** | Option<**bool**> | If true, return the row id as a column called `_rowid` | [optional] diff --git a/rust/lance-namespace-reqwest-client/docs/QueryRequestVector.md b/rust/lance-namespace-reqwest-client/docs/QueryRequestVector.md new file mode 100644 index 000000000..2fef8aa95 --- /dev/null +++ b/rust/lance-namespace-reqwest-client/docs/QueryRequestVector.md @@ -0,0 +1,12 @@ +# QueryRequestVector + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**single_vector** | Option<**Vec**> | Single query vector | [optional] +**multi_vector** | Option<[**Vec>**](Vec.md)> | Multiple query vectors for batch search | [optional] + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/rust/lance-namespace-reqwest-client/src/models/mod.rs b/rust/lance-namespace-reqwest-client/src/models/mod.rs index 72051f662..0e1bdd16c 100644 --- a/rust/lance-namespace-reqwest-client/src/models/mod.rs +++ b/rust/lance-namespace-reqwest-client/src/models/mod.rs @@ -108,6 +108,8 @@ pub mod query_request; pub use self::query_request::QueryRequest; pub mod query_request_full_text_query; pub use self::query_request_full_text_query::QueryRequestFullTextQuery; +pub mod query_request_vector; +pub use self::query_request_vector::QueryRequestVector; pub mod register_table_request; pub use self::register_table_request::RegisterTableRequest; pub mod register_table_response; diff --git a/rust/lance-namespace-reqwest-client/src/models/query_request.rs b/rust/lance-namespace-reqwest-client/src/models/query_request.rs index 84a386d73..67181a511 100644 --- a/rust/lance-namespace-reqwest-client/src/models/query_request.rs +++ b/rust/lance-namespace-reqwest-client/src/models/query_request.rs @@ -58,9 +58,8 @@ pub struct QueryRequest { /// Upper bound for search #[serde(rename = "upper_bound", default, with = "::serde_with::rust::double_option", skip_serializing_if = "Option::is_none")] pub upper_bound: Option>, - /// Query vector for similarity search (single vector only) #[serde(rename = "vector")] - pub vector: Vec, + pub vector: Box, /// Name of the vector column to search #[serde(rename = "vector_column", default, with = "::serde_with::rust::double_option", skip_serializing_if = "Option::is_none")] pub vector_column: Option>, @@ -73,7 +72,7 @@ pub struct QueryRequest { } impl QueryRequest { - pub fn new(name: String, namespace: Vec, k: i32, vector: Vec) -> QueryRequest { + pub fn new(name: String, namespace: Vec, k: i32, vector: models::QueryRequestVector) -> QueryRequest { QueryRequest { name, namespace, @@ -91,7 +90,7 @@ impl QueryRequest { prefilter: None, refine_factor: None, upper_bound: None, - vector, + vector: Box::new(vector), vector_column: None, version: None, with_row_id: None, diff --git a/rust/lance-namespace-reqwest-client/src/models/query_request_vector.rs b/rust/lance-namespace-reqwest-client/src/models/query_request_vector.rs new file mode 100644 index 000000000..b19eac565 --- /dev/null +++ b/rust/lance-namespace-reqwest-client/src/models/query_request_vector.rs @@ -0,0 +1,34 @@ +/* + * Lance REST Namespace Specification + * + * This OpenAPI specification is a part of the Lance namespace specification. It contains 2 parts: The `components/schemas`, `components/responses`, `components/examples` sections define the request and response shape for each operation in a Lance Namespace across all implementations. See https://lancedb.github.io/lance-namespace/spec/operations for more details. The `servers`, `security`, `paths`, `components/parameters` sections are for the Lance REST Namespace implementation, which defines a complete REST server that can work with Lance datasets. See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details. + * + * The version of the OpenAPI document: 0.0.1 + * + * Generated by: https://openapi-generator.tech + */ + +use crate::models; +use serde::{Deserialize, Serialize}; + +/// QueryRequestVector : Query vector(s) for similarity search. Provide either single_vector or multi_vector, not both. +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +pub struct QueryRequestVector { + /// Single query vector + #[serde(rename = "single_vector", skip_serializing_if = "Option::is_none")] + pub single_vector: Option>, + /// Multiple query vectors for batch search + #[serde(rename = "multi_vector", skip_serializing_if = "Option::is_none")] + pub multi_vector: Option>>, +} + +impl QueryRequestVector { + /// Query vector(s) for similarity search. Provide either single_vector or multi_vector, not both. + pub fn new() -> QueryRequestVector { + QueryRequestVector { + single_vector: None, + multi_vector: None, + } + } +} + From 5913db277a87446cfc4ede42f39bd957e7c3af89 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sun, 20 Jul 2025 11:23:32 -0700 Subject: [PATCH 04/10] Refactor tests --- docs/src/user-guide/java-sdk.md | 7 +- .../lance/namespace/RestTableApiTest.java | 1492 ----------------- .../namespace/test/BaseNamespaceTest.java | 124 ++ .../lance/namespace/test/index/IndexTest.java | 296 ++++ .../test/query/FullTextSearchTest.java | 450 +++++ .../lance/namespace/test/query/QueryTest.java | 267 +++ .../test/table/TableLifecycleTest.java | 217 +++ .../test/table/TableMergeInsertTest.java | 199 +++ .../test/table/TableUpdateDeleteTest.java | 151 ++ .../namespace/test/utils/ArrowTestUtils.java | 366 ++++ .../lance/namespace/test/utils/TestUtils.java | 144 ++ 11 files changed, 2220 insertions(+), 1493 deletions(-) delete mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/RestTableApiTest.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/BaseNamespaceTest.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/QueryTest.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableLifecycleTest.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableMergeInsertTest.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableUpdateDeleteTest.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/ArrowTestUtils.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java diff --git a/docs/src/user-guide/java-sdk.md b/docs/src/user-guide/java-sdk.md index f028fd1c0..6322662f0 100644 --- a/docs/src/user-guide/java-sdk.md +++ b/docs/src/user-guide/java-sdk.md @@ -180,6 +180,9 @@ try (BufferAllocator allocator = new RootAllocator(); Query results are returned in Arrow File format. Use `ArrowFileReader` to read the results. +!!! important "Column Selection Required" + When querying a table, you MUST specify which columns to return using `setColumns()`. If no columns are specified, the query will fail with an error: "no columns were selected and with_row_id is false, there is nothing to scan". + #### Vector Search ```java @@ -207,7 +210,7 @@ for (int i = 0; i < 128; i++) { queryRequest.setVector(queryVector); queryRequest.setK(5); // Get top 5 results -// Specify columns to return +// REQUIRED: Specify columns to return queryRequest.setColumns(Arrays.asList("id", "name", "embedding")); // Execute query @@ -261,6 +264,7 @@ filterOnlyQuery.setName("my_table"); filterOnlyQuery.setK(10); filterOnlyQuery.setFilter("category = 'electronics' AND price < 500"); filterOnlyQuery.setFastSearch(true); +filterOnlyQuery.setColumns(Arrays.asList("id", "name", "category", "price")); // Required! // Vector search with SQL filter QueryRequest vectorWithFilter = new QueryRequest(); @@ -269,6 +273,7 @@ vectorWithFilter.setVector(queryVector); vectorWithFilter.setK(10); vectorWithFilter.setFilter("status = 'active' AND price < 1000"); vectorWithFilter.setFastSearch(true); +vectorWithFilter.setColumns(Arrays.asList("id", "name", "status", "price")); // Required! ``` ##### Prefilter vs Postfilter diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/RestTableApiTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/RestTableApiTest.java deleted file mode 100644 index 338539c06..000000000 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/RestTableApiTest.java +++ /dev/null @@ -1,1492 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.lancedb.lance.namespace; - -import com.lancedb.lance.namespace.client.apache.ApiClient; -import com.lancedb.lance.namespace.model.*; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.Float4Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.complex.FixedSizeListVector; -import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.ArrowStreamWriter; -import org.apache.arrow.vector.types.FloatingPointPrecision; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.SeekableByteChannel; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -/** Test for Lance REST Table API using environment variables for configuration */ -public class RestTableApiTest { - - // Configuration from environment variables - private static String DATABASE; - private static String API_KEY; - private static String HOST_OVERRIDE; - private static String REGION; - - // Test data - private String testCreateTableName; - - private LanceRestNamespace namespace; - private BufferAllocator allocator; - - @BeforeAll - public static void setUpClass() { - // Get configuration from environment variables - // The top level folder name, e.g. s3://my-bucket/my-database - // LANCEDB_DB is the 'my-database' - DATABASE = System.getenv("LANCEDB_DB"); - API_KEY = System.getenv("LANCEDB_API_KEY"); - // e.g. http://localhost:10024 - HOST_OVERRIDE = System.getenv("LANCEDB_HOST_OVERRIDE"); - REGION = System.getenv("LANCEDB_REGION"); - - // Default values if not set - if (REGION == null) { - REGION = "us-east-1"; - } - - if (DATABASE != null && API_KEY != null) { - System.out.println("Using configuration:"); - System.out.println(" Database: " + DATABASE); - System.out.println(" Region: " + REGION); - System.out.println(" Host Override: " + (HOST_OVERRIDE != null ? HOST_OVERRIDE : "none")); - } - } - - @BeforeEach - public void setUp() { - // Only initialize if required environment variables are set - // TODO add required environment variables as github secrets - if (DATABASE != null && API_KEY != null) { - namespace = initializeClient(); - allocator = new RootAllocator(); - // Generate unique table name for each test run - testCreateTableName = - "test_table_" + UUID.randomUUID().toString().replace("-", "_").substring(0, 8); - } - } - - @Test - public void testTableLifecycle() throws IOException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("=== Test: Table Lifecycle ==="); - - // Create table with 3 rows using helper - System.out.println("\n--- Creating table ---"); - CreateTableResponse createResponse = createTableHelper(testCreateTableName, 3); - assertNotNull(createResponse, "Create response should not be null"); - - // Test count rows - System.out.println("\n--- Testing count rows ---"); - CountRowsRequest countRequest = new CountRowsRequest(); - countRequest.setName(testCreateTableName); - - Long countResponse = namespace.countRows(countRequest); - assertNotNull(countResponse, "Count rows response should not be null"); - assertEquals(3, countResponse.longValue(), "Row count should match expected number"); - System.out.println("✓ Count rows verified: " + countResponse); - - // Test describe table - System.out.println("\n--- Testing describe table ---"); - DescribeTableRequest describeRequest = new DescribeTableRequest(); - describeRequest.setName(testCreateTableName); - - DescribeTableResponse describeResponse = namespace.describeTable(describeRequest); - assertNotNull(describeResponse, "Describe response should not be null"); - assertEquals( - testCreateTableName, describeResponse.getTable(), "Table name should match in describe"); - assertNotNull(describeResponse.getSchema(), "Schema should not be null"); - assertNotNull(describeResponse.getStats(), "Stats should not be null"); - - // Verify schema - JsonSchema responseSchema = describeResponse.getSchema(); - assertNotNull(responseSchema, "Schema object should not be null"); - assertNotNull(responseSchema.getFields(), "Schema fields should not be null"); - assertEquals(3, responseSchema.getFields().size(), "Schema should have 3 fields"); - - List fieldNames = - responseSchema.getFields().stream() - .map(JsonField::getName) - .collect(java.util.stream.Collectors.toList()); - assertTrue(fieldNames.contains("id"), "Schema should contain 'id' field"); - assertTrue(fieldNames.contains("name"), "Schema should contain 'name' field"); - assertTrue(fieldNames.contains("embedding"), "Schema should contain 'embedding' field"); - System.out.println("✓ Table schema verified with fields: " + fieldNames); - - // Verify version and stats - assertNotNull(describeResponse.getVersion(), "Version should not be null"); - assertTrue(describeResponse.getVersion() >= 1, "Version should be at least 1 for new table"); - assertTrue( - describeResponse.getStats().getNumFragments() >= 0, - "Number of fragments should be non-negative"); - System.out.println("✓ Table version: " + describeResponse.getVersion()); - System.out.println("✓ Table fragments: " + describeResponse.getStats().getNumFragments()); - - // Test insert table - System.out.println("\n--- Testing insert table ---"); - // Insert 2 more rows (total should be 5) - InsertTableResponse insertResponse = insertHelper(testCreateTableName, 2, "append", 5); - - // Insert 3 more rows (total should be 8) - System.out.println("\n--- Testing second insert ---"); - InsertTableResponse secondInsertResponse = insertHelper(testCreateTableName, 3, "append", 8); - - // Test drop table using helper - System.out.println("\n--- Testing drop table ---"); - DropTableResponse dropResponse = dropTableHelper(testCreateTableName); - assertNotNull(dropResponse, "Drop table response should not be null"); - - // Verify table was dropped - System.out.println("\n--- Verifying table was dropped ---"); - try { - DescribeTableRequest verifyDropRequest = new DescribeTableRequest(); - verifyDropRequest.setName(testCreateTableName); - namespace.describeTable(verifyDropRequest); - fail("Expected exception when describing dropped table"); - } catch (LanceNamespaceException e) { - assertEquals(404, e.getCode(), "Should get 404 error code for non-existent table"); - System.out.println("✓ Confirmed table no longer exists (404 error code)"); - } - - System.out.println("\n✓ Table lifecycle test passed!"); - } - - /** - * Helper method to create a table with the specified number of rows - * - * @param tableName The name of the table to create - * @param numRows The number of rows to create in the table - * @return CreateTableResponse from the server - * @throws IOException if there's an error creating the table - */ - private CreateTableResponse createTableHelper(String tableName, int numRows) throws IOException { - // Create Arrow schema - Field idField = new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null); - Field nameField = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); - Field embeddingField = - new Field( - "embedding", - FieldType.nullable(new ArrowType.FixedSizeList(128)), - Arrays.asList( - new Field( - "item", - FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), - null))); - - Schema schema = new Schema(Arrays.asList(idField, nameField, embeddingField)); - - try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { - IntVector idVector = (IntVector) root.getVector("id"); - VarCharVector nameVector = (VarCharVector) root.getVector("name"); - FixedSizeListVector vectorVector = (FixedSizeListVector) root.getVector("embedding"); - - root.setRowCount(numRows); - - // Sample names for test data - String[] names = { - "Alice", "Bob", "Charlie", "David", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack" - }; - - // Populate data for specified number of rows - for (int i = 0; i < numRows; i++) { - idVector.setSafe(i, i + 1); - nameVector.setSafe(i, names[i % names.length].getBytes(StandardCharsets.UTF_8)); - } - - // Populate vector field with dummy data - Float4Vector dataVector = (Float4Vector) vectorVector.getDataVector(); - vectorVector.allocateNew(); - - // Create 128-dimensional vectors for each row - for (int row = 0; row < numRows; row++) { - vectorVector.setNotNull(row); - for (int dim = 0; dim < 128; dim++) { - int index = row * 128 + dim; - dataVector.setSafe(index, (float) (Math.random() * 10.0)); // Random values 0-10 - } - } - - // Mark vectors as populated - idVector.setValueCount(numRows); - nameVector.setValueCount(numRows); - dataVector.setValueCount(numRows * 128); // numRows * 128 dimensions - vectorVector.setValueCount(numRows); - - // Serialize to Arrow IPC format - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(out))) { - writer.start(); - writer.writeBatch(); - writer.end(); - } - - byte[] arrowIpcData = out.toByteArray(); - System.out.println("Arrow IPC data size: " + arrowIpcData.length + " bytes"); - - // Create table using Arrow IPC data - CreateTableResponse response = namespace.createTable(tableName, arrowIpcData); - System.out.println( - "✓ Table created successfully: " + tableName + " with " + numRows + " rows"); - - return response; - } - } - - /** - * Helper method to drop a table - * - * @param tableName The name of the table to drop - * @return DropTableResponse from the server - */ - private DropTableResponse dropTableHelper(String tableName) { - DropTableRequest dropRequest = new DropTableRequest(); - dropRequest.setName(tableName); - - DropTableResponse response = namespace.dropTable(dropRequest); - System.out.println("✓ Table dropped successfully: " + tableName); - - return response; - } - - /** - * Helper method to insert data into a table - * - * @param tableName The name of the table to insert into - * @param numRows The number of rows to insert - * @param mode The insert mode ("append" or "overwrite") - * @return InsertTableResponse from the server - * @throws IOException if there's an error creating the Arrow data - */ - private InsertTableResponse insertTableHelper(String tableName, int numRows, String mode) - throws IOException { - // Create Arrow schema (same as createTableHelper) - Field idField = new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null); - Field nameField = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); - Field vectorField = - new Field( - "embedding", - FieldType.nullable(new ArrowType.FixedSizeList(128)), - Arrays.asList( - new Field( - "item", - FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), - null))); - - Schema schema = new Schema(Arrays.asList(idField, nameField, vectorField)); - - try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { - IntVector idVector = (IntVector) root.getVector("id"); - VarCharVector nameVector = (VarCharVector) root.getVector("name"); - FixedSizeListVector vectorVector = (FixedSizeListVector) root.getVector("embedding"); - - root.setRowCount(numRows); - - // Sample names for test data - String[] names = { - "Liam", "Emma", "Noah", "Olivia", "William", "Ava", "James", "Isabella", "Logan", "Sophia" - }; - - // Populate data for specified number of rows - // Start IDs from 1000 to differentiate from initial data - for (int i = 0; i < numRows; i++) { - idVector.setSafe(i, 1000 + i); - nameVector.setSafe(i, names[i % names.length].getBytes(StandardCharsets.UTF_8)); - } - - // Populate vector field with dummy data - Float4Vector dataVector = (Float4Vector) vectorVector.getDataVector(); - vectorVector.allocateNew(); - - // Create 128-dimensional vectors for each row - for (int row = 0; row < numRows; row++) { - vectorVector.setNotNull(row); - for (int dim = 0; dim < 128; dim++) { - int index = row * 128 + dim; - dataVector.setSafe(index, (float) (Math.random() * 10.0)); // Random values 0-10 - } - } - - // Mark vectors as populated - idVector.setValueCount(numRows); - nameVector.setValueCount(numRows); - dataVector.setValueCount(numRows * 128); // numRows * 128 dimensions - vectorVector.setValueCount(numRows); - - // Serialize to Arrow IPC format - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(out))) { - writer.start(); - writer.writeBatch(); - writer.end(); - } - - byte[] arrowIpcData = out.toByteArray(); - System.out.println("Arrow IPC data size for insert: " + arrowIpcData.length + " bytes"); - - // Insert data using Arrow IPC data - InsertTableResponse response = namespace.insertTable(tableName, arrowIpcData, mode); - System.out.println( - "✓ Data inserted successfully: " - + tableName - + " with " - + numRows - + " rows (" - + mode - + " mode)"); - - return response; - } - } - - /** - * Helper method to insert data and verify the operation - * - * @param tableName The name of the table to insert into - * @param numRows The number of rows to insert - * @param mode The insert mode ("append" or "overwrite") - * @param expectedTotalRows The expected total row count after insert - * @return InsertTableResponse from the server - * @throws IOException if there's an error creating the Arrow data - */ - private InsertTableResponse insertHelper( - String tableName, int numRows, String mode, long expectedTotalRows) throws IOException { - // Insert data - InsertTableResponse response = insertTableHelper(tableName, numRows, mode); - assertNotNull(response, "Insert response should not be null"); - assertNotNull(response.getVersion(), "Insert response version should not be null"); - System.out.println("✓ Inserted " + numRows + " rows, new version: " + response.getVersion()); - - // Verify row count - CountRowsRequest countRequest = new CountRowsRequest(); - countRequest.setName(tableName); - Long actualCount = namespace.countRows(countRequest); - assertEquals( - expectedTotalRows, - actualCount.longValue(), - "Row count should be " + expectedTotalRows + " after insert"); - System.out.println("✓ Verified row count: " + actualCount); - - return response; - } - - @Test - public void testQueryTable() throws IOException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - String queryTableName = - "test_query_table_" + UUID.randomUUID().toString().replace("-", "_").substring(0, 8); - - CreateTableResponse createResponse = createTableHelper(queryTableName, 10); - assertNotNull(createResponse, "Create response should not be null"); - QueryRequest queryRequest = new QueryRequest(); - queryRequest.setName(queryTableName); - List queryVector = new ArrayList<>(); - for (int i = 0; i < 128; i++) { - queryVector.add((float) (i % 10)); // Use pattern 0-9 repeating - } - queryRequest.setVector(queryVector); - queryRequest.setK(5); // Request top 5 results - - List columns = Arrays.asList("id", "name", "embedding"); - queryRequest.setColumns(columns); - - byte[] queryResult; - queryResult = namespace.queryTable(queryRequest); - assertNotNull(queryResult, "Query result should not be null"); - - // Read and verify the Arrow IPC result - try (BufferAllocator verifyAllocator = new RootAllocator()) { - // Arrow file format - use ArrowFileReader - ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(queryResult); - - try (ArrowFileReader reader = new ArrowFileReader(channel, verifyAllocator)) { - - // Get schema - Schema resultSchema = reader.getVectorSchemaRoot().getSchema(); - List resultColumns = new ArrayList<>(); - for (Field field : resultSchema.getFields()) { - resultColumns.add(field.getName()); - } - System.out.println("Result columns: " + resultColumns); - - // Verify columns - assertTrue(resultColumns.contains("id"), "Result should contain 'id' column"); - assertTrue(resultColumns.contains("name"), "Result should contain 'name' column"); - - // Count total rows - int totalRows = 0; - for (int i = 0; i < reader.getRecordBlocks().size(); i++) { - reader.loadRecordBatch(reader.getRecordBlocks().get(i)); - VectorSchemaRoot root = reader.getVectorSchemaRoot(); - totalRows += root.getRowCount(); - } - - System.out.println("Query returned " + totalRows + " rows"); - assertTrue(totalRows > 0, "Query should return at least some rows"); - } - } - DropTableResponse dropResponse = dropTableHelper(queryTableName); - assertNotNull(dropResponse, "Drop table response should not be null"); - } - - @Test - public void testAdvancedQueryFeatures() throws IOException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("\n=== Test: Advanced Query Features ==="); - String tableName = "test_advanced_query_" + System.currentTimeMillis(); - - try { - // Create table with more rows for better testing - CreateTableResponse createResponse = createTableHelper(tableName, 100); - assertNotNull(createResponse, "Create response should not be null"); - - // Test 1: Query with filter - System.out.println("\n--- Test 1: Query with filter ---"); - QueryRequest filterQuery = new QueryRequest(); - filterQuery.setName(tableName); - filterQuery.setK(10); - filterQuery.setFilter("id > 50"); // SQL filter - // Don't set vector for filter-only queries - - byte[] filterResult = namespace.queryTable(filterQuery); - assertNotNull(filterResult, "Filter query result should not be null"); - verifyQueryResultRowCount(filterResult, 10, "Filter query"); - - // Test 2: Query with prefilter - System.out.println("\n--- Test 4: Query with prefilter ---"); - QueryRequest prefilterQuery = new QueryRequest(); - prefilterQuery.setName(tableName); - prefilterQuery.setK(5); - prefilterQuery.setPrefilter(true); - prefilterQuery.setFilter("id < 20"); - - byte[] prefilterResult = namespace.queryTable(prefilterQuery); - assertNotNull(prefilterResult, "Prefilter query result should not be null"); - - // Test 3: Query with fast_search true - System.out.println("\n--- Test 5: Query with fast_search=true ---"); - QueryRequest fastSearchQuery = new QueryRequest(); - fastSearchQuery.setName(tableName); - List fastSearchVector = new ArrayList<>(); - for (int i = 0; i < 128; i++) { - fastSearchVector.add((float) (i % 10)); - } - fastSearchQuery.setVector(fastSearchVector); - fastSearchQuery.setK(10); - fastSearchQuery.setFastSearch(true); - - byte[] fastSearchResult = namespace.queryTable(fastSearchQuery); - assertNotNull(fastSearchResult, "Fast search query result should not be null"); - verifyQueryResultRowCount(fastSearchResult, 10, "Fast search query"); - - // Test 4: Query with fast_search false - System.out.println("\n--- Test 6: Query with fast_search=false ---"); - QueryRequest noFastSearchQuery = new QueryRequest(); - noFastSearchQuery.setName(tableName); - List noFastSearchVector = new ArrayList<>(); - for (int i = 0; i < 128; i++) { - noFastSearchVector.add((float) (i % 10)); - } - noFastSearchQuery.setVector(noFastSearchVector); - noFastSearchQuery.setK(10); - noFastSearchQuery.setFastSearch(false); - - byte[] noFastSearchResult = namespace.queryTable(noFastSearchQuery); - assertNotNull(noFastSearchResult, "No fast search query result should not be null"); - verifyQueryResultRowCount(noFastSearchResult, 10, "No fast search query"); - - } finally { - dropTableHelper(tableName); - } - } - - @Test - public void testDescribeTableWithVersion() throws IOException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("\n=== Test: Describe Table With Version ==="); - String tableName = "test_describe_version_" + System.currentTimeMillis(); - - try { - // Create table - CreateTableResponse createResponse = createTableHelper(tableName, 5); - assertNotNull(createResponse, "Create response should not be null"); - - // Get initial version - DescribeTableRequest describeV1 = new DescribeTableRequest(); - describeV1.setName(tableName); - DescribeTableResponse v1Response = namespace.describeTable(describeV1); - Long version1 = v1Response.getVersion(); - System.out.println("Initial version: " + version1); - - // Insert more data to create new version - insertTableHelper(tableName, 5, "append"); - - // Describe current version - DescribeTableRequest describeCurrent = new DescribeTableRequest(); - describeCurrent.setName(tableName); - DescribeTableResponse currentResponse = namespace.describeTable(describeCurrent); - Long currentVersion = currentResponse.getVersion(); - System.out.println("Current version after insert: " + currentVersion); - assertTrue(currentVersion > version1, "Version should increase after insert"); - - // Describe specific older version - DescribeTableRequest describeOldVersion = new DescribeTableRequest(); - describeOldVersion.setName(tableName); - describeOldVersion.setVersion(version1); - DescribeTableResponse oldVersionResponse = namespace.describeTable(describeOldVersion); - - assertEquals(version1, oldVersionResponse.getVersion(), "Should return requested version"); - - // Verify nested structures in response - assertNotNull(oldVersionResponse.getSchema(), "Schema should not be null"); - assertNotNull(oldVersionResponse.getSchema().getFields(), "Schema fields should not be null"); - - // Check JsonField structure - for (JsonField field : oldVersionResponse.getSchema().getFields()) { - assertNotNull(field.getName(), "Field name should not be null"); - assertNotNull(field.getType(), "Field type should not be null"); - assertNotNull(field.getNullable(), "Field nullable should not be null"); - - // Check JsonDataType structure - JsonDataType dataType = field.getType(); - assertNotNull(dataType.getType(), "Data type name should not be null"); - - // For FixedSizeList (embedding field), check nested fields - if ("embedding".equals(field.getName())) { - assertNotNull(dataType.getFields(), "Embedding field should have nested fields"); - assertFalse(dataType.getFields().isEmpty(), "Embedding field should have item field"); - } - } - - // Verify TableBasicStats structure - TableBasicStats stats = oldVersionResponse.getStats(); - assertNotNull(stats, "Stats should not be null"); - assertNotNull(stats.getNumDeletedRows(), "Num deleted rows should not be null"); - assertNotNull(stats.getNumFragments(), "Num fragments should not be null"); - assertTrue(stats.getNumFragments() >= 0, "Num fragments should be non-negative"); - - System.out.println("✓ Describe table with version tested successfully"); - - } finally { - dropTableHelper(tableName); - } - } - - @Test - public void testFtsQuery() throws IOException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("\n=== Test: Full-Text Search Query ==="); - String tableName = "test_fts_query_" + System.currentTimeMillis(); - - try { - // Create table with text data - CreateTableResponse createResponse = createTextTableHelper(tableName, 50); - assertNotNull(createResponse, "Create response should not be null"); - - // Create FTS index - System.out.println("\n--- Creating FTS index ---"); - CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); - ftsIndexRequest.setName(tableName); - ftsIndexRequest.setColumn("text"); - ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); - - CreateIndexResponse ftsIndexResponse = namespace.createIndex(ftsIndexRequest); - assertNotNull(ftsIndexResponse, "FTS index response should not be null"); - - waitForIndexComplete(tableName, "text_idx", 30); - - // Test 1: Simple string FTS query - System.out.println("\n--- Test 1: Simple string FTS query ---"); - QueryRequest stringFtsQuery = new QueryRequest(); - stringFtsQuery.setName(tableName); - stringFtsQuery.setK(10); - - StringFtsQuery simpleFts = new StringFtsQuery(); - simpleFts.setQuery("document"); - QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); - fullTextQuery.setStringQuery(simpleFts); - stringFtsQuery.setFullTextQuery(fullTextQuery); - - byte[] ftsResult = namespace.queryTable(stringFtsQuery); - assertNotNull(ftsResult, "FTS query result should not be null"); - - // Test 2: FTS with prefilter - System.out.println("\n--- Test 2: FTS with prefilter ---"); - QueryRequest ftsPrefilterQuery = new QueryRequest(); - ftsPrefilterQuery.setName(tableName); - ftsPrefilterQuery.setK(5); - ftsPrefilterQuery.setPrefilter(true); - ftsPrefilterQuery.setFilter("id < 25"); - ftsPrefilterQuery.setFullTextQuery(fullTextQuery); - - byte[] ftsPrefilterResult = namespace.queryTable(ftsPrefilterQuery); - assertNotNull(ftsPrefilterResult, "FTS prefilter query result should not be null"); - - System.out.println("✓ FTS query tests completed successfully"); - - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - fail("Test interrupted"); - } finally { - dropTableHelper(tableName); - } - } - - /** Helper method to create a table with text data for FTS testing */ - private CreateTableResponse createTextTableHelper(String tableName, int numRows) - throws IOException { - // Create Arrow schema with text field - Field idField = new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null); - Field textField = new Field("text", FieldType.nullable(new ArrowType.Utf8()), null); - Field embeddingField = - new Field( - "embedding", - FieldType.nullable(new ArrowType.FixedSizeList(128)), - Arrays.asList( - new Field( - "item", - FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), - null))); - - Schema schema = new Schema(Arrays.asList(idField, textField, embeddingField)); - - try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { - IntVector idVector = (IntVector) root.getVector("id"); - VarCharVector textVector = (VarCharVector) root.getVector("text"); - FixedSizeListVector vectorVector = (FixedSizeListVector) root.getVector("embedding"); - - root.setRowCount(numRows); - - // Sample text patterns for FTS - String[] textTemplates = { - "This is document number %d", - "Sample text for entry %d", - "Test document with id %d", - "Record %d contains important data", - "Entry %d in the database" - }; - - // Populate data - for (int i = 0; i < numRows; i++) { - idVector.setSafe(i, i + 1); - String text = String.format(textTemplates[i % textTemplates.length], i); - textVector.setSafe(i, text.getBytes(StandardCharsets.UTF_8)); - } - - // Populate vector field - Float4Vector dataVector = (Float4Vector) vectorVector.getDataVector(); - vectorVector.allocateNew(); - - for (int row = 0; row < numRows; row++) { - vectorVector.setNotNull(row); - for (int dim = 0; dim < 128; dim++) { - int index = row * 128 + dim; - dataVector.setSafe(index, (float) (Math.random() * 10.0)); - } - } - - idVector.setValueCount(numRows); - textVector.setValueCount(numRows); - dataVector.setValueCount(numRows * 128); - vectorVector.setValueCount(numRows); - - // Serialize to Arrow IPC format - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(out))) { - writer.start(); - writer.writeBatch(); - writer.end(); - } - - byte[] arrowIpcData = out.toByteArray(); - - CreateTableResponse response = namespace.createTable(tableName, arrowIpcData); - System.out.println("✓ Text table created: " + tableName + " with " + numRows + " rows"); - - return response; - } - } - - /** Helper method to wait for index creation */ - private boolean waitForIndex(String tableName, String indexName, int maxSeconds) - throws InterruptedException { - IndexListRequest listRequest = new IndexListRequest(); - listRequest.setName(tableName); - - for (int i = 0; i < maxSeconds; i++) { - IndexListResponse listResponse = namespace.listIndices(listRequest); - if (listResponse.getIndexes() != null - && listResponse.getIndexes().stream() - .anyMatch(idx -> idx.getIndexName().equals(indexName))) { - return true; - } - Thread.sleep(1000); - } - return false; - } - - /** Helper method to wait for index to be fully built with no unindexed rows */ - private boolean waitForIndexComplete(String tableName, String indexName, int maxSeconds) - throws InterruptedException { - IndexListRequest listRequest = new IndexListRequest(); - listRequest.setName(tableName); - - for (int i = 0; i < maxSeconds; i++) { - IndexListResponse listResponse = namespace.listIndices(listRequest); - if (listResponse.getIndexes() != null) { - Optional indexOpt = - listResponse.getIndexes().stream() - .filter(idx -> idx.getIndexName().equals(indexName)) - .findFirst(); - - if (indexOpt.isPresent()) { - // Index exists, now check if it's fully built - IndexStatsRequest statsRequest = new IndexStatsRequest(); - statsRequest.setName(tableName); - - IndexStatsResponse stats = namespace.getIndexStats(statsRequest, indexName); - if (stats != null - && stats.getNumUnindexedRows() != null - && stats.getNumUnindexedRows() == 0) { - System.out.println("✓ Index " + indexName + " is fully built with 0 unindexed rows"); - return true; - } else if (stats != null && stats.getNumUnindexedRows() != null) { - System.out.println( - " Waiting for index... " + stats.getNumUnindexedRows() + " rows remaining"); - } - } - } - Thread.sleep(1000); - } - return false; - } - - /** Helper method to verify query result row count */ - private void verifyQueryResultRowCount(byte[] queryResult, int expectedRows, String testName) - throws IOException { - try (BufferAllocator verifyAllocator = new RootAllocator()) { - ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(queryResult); - try (ArrowFileReader reader = new ArrowFileReader(channel, verifyAllocator)) { - int totalRows = 0; - for (int i = 0; i < reader.getRecordBlocks().size(); i++) { - reader.loadRecordBatch(reader.getRecordBlocks().get(i)); - VectorSchemaRoot root = reader.getVectorSchemaRoot(); - totalRows += root.getRowCount(); - } - assertEquals( - expectedRows, totalRows, testName + " should return " + expectedRows + " rows"); - } - } - } - - @Test - public void testHybridSearch() throws IOException, InterruptedException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("\n=== Test: Hybrid Search (Vector + Full-Text) ==="); - String tableName = "test_hybrid_search_" + System.currentTimeMillis(); - - try { - // Create table with text and vector data - CreateTableResponse createResponse = createTextTableHelper(tableName, 50); - assertNotNull(createResponse, "Create response should not be null"); - - // Create FTS index - System.out.println("\n--- Creating FTS index for hybrid search ---"); - CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); - ftsIndexRequest.setName(tableName); - ftsIndexRequest.setColumn("text"); - ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); - - CreateIndexResponse ftsIndexResponse = namespace.createIndex(ftsIndexRequest); - assertNotNull(ftsIndexResponse, "FTS index response should not be null"); - - waitForIndexComplete(tableName, "text_idx", 30); - - // Test 1: Hybrid search with vector and text - System.out.println("\n--- Test 1: Hybrid search with vector and full-text query ---"); - QueryRequest hybridQuery = new QueryRequest(); - hybridQuery.setName(tableName); - - // Add vector search - List queryVector = new ArrayList<>(); - for (int i = 0; i < 128; i++) { - queryVector.add((float) (i % 10)); - } - hybridQuery.setVector(queryVector); - hybridQuery.setK(10); - - // Add full-text search - StringFtsQuery ftsQuery = new StringFtsQuery(); - ftsQuery.setQuery("document"); - QueryRequestFullTextQuery hybridFullTextQuery = new QueryRequestFullTextQuery(); - hybridFullTextQuery.setStringQuery(ftsQuery); - hybridQuery.setFullTextQuery(hybridFullTextQuery); - - byte[] hybridResult = namespace.queryTable(hybridQuery); - assertNotNull(hybridResult, "Hybrid search result should not be null"); - System.out.println("✓ Hybrid search completed successfully"); - - // Test 2: Hybrid search with filter - System.out.println("\n--- Test 2: Hybrid search with vector, text, and filter ---"); - QueryRequest hybridFilterQuery = new QueryRequest(); - hybridFilterQuery.setName(tableName); - - // Add vector search - List queryVector2 = new ArrayList<>(); - for (int i = 0; i < 128; i++) { - queryVector2.add((float) (i % 5)); - } - hybridFilterQuery.setVector(queryVector2); - hybridFilterQuery.setK(5); - - // Add full-text search - StringFtsQuery ftsQuery2 = new StringFtsQuery(); - ftsQuery2.setQuery("entry"); - QueryRequestFullTextQuery hybridFilterFullTextQuery = new QueryRequestFullTextQuery(); - hybridFilterFullTextQuery.setStringQuery(ftsQuery2); - hybridFilterQuery.setFullTextQuery(hybridFilterFullTextQuery); - - // Add filter - hybridFilterQuery.setFilter("id < 30"); - - byte[] hybridFilterResult = namespace.queryTable(hybridFilterQuery); - assertNotNull(hybridFilterResult, "Hybrid search with filter result should not be null"); - System.out.println("✓ Hybrid search with filter completed successfully"); - - // Test 3: Hybrid search with column selection - System.out.println("\n--- Test 3: Hybrid search with specific columns ---"); - QueryRequest hybridColumnsQuery = new QueryRequest(); - hybridColumnsQuery.setName(tableName); - - // Add vector search - List queryVector3 = new ArrayList<>(); - for (int i = 0; i < 128; i++) { - queryVector3.add((float) Math.random()); - } - hybridColumnsQuery.setVector(queryVector3); - hybridColumnsQuery.setK(8); - - // Add full-text search with columns - StringFtsQuery ftsQuery3 = new StringFtsQuery(); - ftsQuery3.setQuery("test"); - ftsQuery3.setColumns(Arrays.asList("text")); // Search only in text column - QueryRequestFullTextQuery hybridColumnsFullTextQuery = new QueryRequestFullTextQuery(); - hybridColumnsFullTextQuery.setStringQuery(ftsQuery3); - hybridColumnsQuery.setFullTextQuery(hybridColumnsFullTextQuery); - - // Return specific columns - hybridColumnsQuery.setColumns(Arrays.asList("id", "text")); - - byte[] hybridColumnsResult = namespace.queryTable(hybridColumnsQuery); - assertNotNull(hybridColumnsResult, "Hybrid search with columns result should not be null"); - verifyQueryResultRowCount(hybridColumnsResult, 8, "Hybrid search with columns"); - - System.out.println("✓ All hybrid search tests completed successfully"); - - } finally { - dropTableHelper(tableName); - } - } - - private LanceRestNamespace initializeClient() { - Map config = new HashMap<>(); - config.put("headers.x-lancedb-database", DATABASE); - config.put("headers.x-api-key", API_KEY); - - if (HOST_OVERRIDE != null) { - config.put("host_override", HOST_OVERRIDE); - } - if (REGION != null) { - config.put("region", REGION); - } - - ApiClient apiClient = new ApiClient(); - - // Set base URL based on configuration - String baseUrl; - if (HOST_OVERRIDE != null) { - baseUrl = HOST_OVERRIDE; - } else { - baseUrl = String.format("https://%s.%s.api.lancedb.com", DATABASE, REGION); - } - apiClient.setBasePath(baseUrl); - - System.out.println("Initialized client with base URL: " + baseUrl); - - return new LanceRestNamespace(apiClient, config); - } - - @Test - public void testCreateIndex() throws IOException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("\n=== Test: Create Index ==="); - String indexTableName = "test_index_table_" + System.currentTimeMillis(); - - // Step 1: Create table with 300 rows - System.out.println("\n--- Step 1: Creating table with 300 rows ---"); - CreateTableResponse createResponse = createTableHelper(indexTableName, 300); - assertNotNull(createResponse, "Create table response should not be null"); - - try { - // Step 2: List indices before creating index (should be empty) - System.out.println("\n--- Step 2: Listing indices before index creation ---"); - IndexListRequest listRequestBefore = new IndexListRequest(); - listRequestBefore.setName(indexTableName); - IndexListResponse listResponseBefore = namespace.listIndices(listRequestBefore); - assertNotNull(listResponseBefore, "List indices response should not be null"); - assertNotNull(listResponseBefore.getIndexes(), "Indexes list should not be null"); - assertEquals( - 0, listResponseBefore.getIndexes().size(), "Should have no indices before creation"); - System.out.println("✓ Confirmed no indices exist before creation"); - - // Step 3: Create vector index - System.out.println("\n--- Step 3: Creating vector index ---"); - CreateIndexRequest indexRequest = new CreateIndexRequest(); - indexRequest.setName(indexTableName); - indexRequest.setColumn("embedding"); - indexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.IVF_PQ); - indexRequest.setMetricType(CreateIndexRequest.MetricTypeEnum.L2); - - CreateIndexResponse indexResponse = namespace.createIndex(indexRequest); - assertNotNull(indexResponse, "Create index response should not be null"); - System.out.println("✓ Index creation request submitted successfully"); - - // Step 4: Wait for index creation completion (with timeout) - System.out.println("\n--- Step 4: Waiting for index creation to complete ---"); - IndexListRequest listRequestAfter = new IndexListRequest(); - listRequestAfter.setName(indexTableName); - - boolean indexFound = false; - int attempts = 0; - int maxAttempts = 60; // 60 attempts = ~60 seconds with 1 second intervals - - while (!indexFound && attempts < maxAttempts) { - attempts++; - try { - Thread.sleep(1000); // Wait 1 second between attempts - - IndexListResponse listResponseAfter = namespace.listIndices(listRequestAfter); - assertNotNull(listResponseAfter, "List indices response should not be null"); - - if (listResponseAfter.getIndexes() != null && !listResponseAfter.getIndexes().isEmpty()) { - indexFound = true; - assertEquals(1, listResponseAfter.getIndexes().size(), "Should have exactly one index"); - assertTrue( - listResponseAfter.getIndexes().get(0).getIndexName().equals("embedding_idx")); - } else { - System.out.println( - "⏳ Attempt " + attempts + "/" + maxAttempts + " - Index not ready yet..."); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - fail("Test interrupted while waiting for index creation"); - } - } - assertTrue(indexFound, "Index should be found after creation"); - - // Step 5: Get index stats - System.out.println("\n--- Step 5: Getting index stats for embedding_idx ---"); - IndexStatsRequest statsRequest = new IndexStatsRequest(); - statsRequest.setName(indexTableName); - - IndexStatsResponse statsResponse = namespace.getIndexStats(statsRequest, "embedding_idx"); - assertNotNull(statsResponse, "Index stats response should not be null"); - assertEquals("IVF_PQ", statsResponse.getIndexType()); - assertEquals("l2", statsResponse.getDistanceType()); - assertEquals(300, statsResponse.getNumIndexedRows().intValue()); - assertEquals(0, statsResponse.getNumUnindexedRows().intValue()); - } finally { - DropTableResponse dropResponse = dropTableHelper(indexTableName); - } - } - - @Test - public void testCreateScalarIndex() throws IOException, InterruptedException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("\n=== Test: Create Scalar Index ==="); - String scalarIndexTableName = "test_scalar_index_table_" + System.currentTimeMillis(); - - try { - // Step 1: Create table with 300 rows - System.out.println("\n--- Step 1: Creating table with 300 rows ---"); - CreateTableResponse createResponse = createTableHelper(scalarIndexTableName, 300); - assertNotNull(createResponse, "Create table response should not be null"); - - // Step 2: List indices before creating index (should be empty) - System.out.println("\n--- Step 2: Listing indices before index creation ---"); - IndexListRequest listRequestBefore = new IndexListRequest(); - listRequestBefore.setName(scalarIndexTableName); - IndexListResponse listResponseBefore = namespace.listIndices(listRequestBefore); - assertNotNull(listResponseBefore, "List indices response should not be null"); - assertNotNull(listResponseBefore.getIndexes(), "Indexes list should not be null"); - assertEquals( - 0, listResponseBefore.getIndexes().size(), "Should have no indices before creation"); - System.out.println("✓ Confirmed no indices exist before creation"); - - // Step 3: Create scalar index on name column - System.out.println("\n--- Step 3: Creating scalar index ---"); - CreateIndexRequest scalarIndexRequest = new CreateIndexRequest(); - scalarIndexRequest.setName(scalarIndexTableName); - scalarIndexRequest.setColumn("name"); - scalarIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.BITMAP); - - CreateIndexResponse scalarIndexResponse = namespace.createScalarIndex(scalarIndexRequest); - assertNotNull(scalarIndexResponse, "Create scalar index response should not be null"); - - // Step 4: Wait for index creation completion (with timeout) - System.out.println("\n--- Step 4: Waiting for scalar index creation to complete ---"); - IndexListRequest listRequestAfter = new IndexListRequest(); - listRequestAfter.setName(scalarIndexTableName); - - boolean indexFound = false; - int attempts = 0; - int maxAttempts = 60; // 60 attempts = ~60 seconds with 1 second intervals - - while (!indexFound && attempts < maxAttempts) { - attempts++; - try { - Thread.sleep(1000); // Wait 1 second between attempts - - IndexListResponse listResponseAfter = namespace.listIndices(listRequestAfter); - assertNotNull(listResponseAfter, "List indices response should not be null"); - - if (listResponseAfter.getIndexes() != null && !listResponseAfter.getIndexes().isEmpty()) { - indexFound = true; - assertEquals(1, listResponseAfter.getIndexes().size(), "Should have exactly one index"); - assertTrue( - listResponseAfter.getIndexes().get(0).getColumns().contains("name"), - "Index should be on name column"); - } else { - System.out.println( - "⏳ Attempt " + attempts + "/" + maxAttempts + " - Scalar index not ready yet..."); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - fail("Test interrupted while waiting for scalar index creation"); - } - } - - assertTrue(indexFound, "Index should be found after creation"); - - IndexStatsRequest statsRequest = new IndexStatsRequest(); - statsRequest.setName(scalarIndexTableName); - - IndexStatsResponse statsResponse = namespace.getIndexStats(statsRequest, "name_idx"); - assertNotNull(statsResponse, "Index stats response should not be null"); - assertEquals("BITMAP", statsResponse.getIndexType()); - assertEquals(300, statsResponse.getNumIndexedRows().intValue()); - assertEquals(0, statsResponse.getNumUnindexedRows().intValue()); - } finally { - DropTableResponse dropResponse = dropTableHelper(scalarIndexTableName); - } - } - - @Test - public void testUpdateTable() throws IOException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("\n=== Test: Update Table ==="); - String updateTableName = "test_update_table_" + System.currentTimeMillis(); - - try { - CreateTableResponse createResponse = createTableHelper(updateTableName, 3); - assertNotNull(createResponse, "Create table response should not be null"); - // Id: 1 2 3 - System.out.println("\n--- Step 2: Updating all rows (id = id + 1) ---"); - UpdateTableRequest updateRequest = new UpdateTableRequest(); - updateRequest.setName(updateTableName); - updateRequest.setNamespace(new ArrayList<>()); - List> updates = new ArrayList<>(); - updates.add(Arrays.asList("id", "id + 1")); - updateRequest.setUpdates(updates); - // Id: 2 3 4 - UpdateTableResponse updateResponse = namespace.updateTable(updateRequest); - assertNotNull(updateResponse, "Update response should not be null"); - assertEquals(3, updateResponse.getUpdatedRows().longValue(), "Should have updated 3 rows"); - - System.out.println("\n--- Step 3: Updating rows with predicate (id > 2) ---"); - UpdateTableRequest predicateUpdateRequest = new UpdateTableRequest(); - predicateUpdateRequest.setName(updateTableName); - predicateUpdateRequest.setNamespace(new ArrayList<>()); - predicateUpdateRequest.setPredicate("id > 2"); - List> predicateUpdates = new ArrayList<>(); - predicateUpdates.add(Arrays.asList("id", "id + 10")); - predicateUpdateRequest.setUpdates(predicateUpdates); - // Id: 2 13 14 - - UpdateTableResponse predicateUpdateResponse = namespace.updateTable(predicateUpdateRequest); - assertNotNull(predicateUpdateResponse, "Predicate update response should not be null"); - assertEquals( - 2, predicateUpdateResponse.getUpdatedRows().longValue(), "Should have updated 2 rows"); - - // Use query to verify final state - QueryRequest queryRequest = new QueryRequest(); - queryRequest.setName(updateTableName); - queryRequest.setK(3); - // Don't set columns or vector for simple queries - - byte[] queryResult; - queryResult = namespace.queryTable(queryRequest); - assertNotNull(queryResult, "Query result should not be null"); - - try (BufferAllocator verifyAllocator = new RootAllocator()) { - ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(queryResult); - - try (ArrowFileReader reader = new ArrowFileReader(channel, verifyAllocator)) { - List idValues = new ArrayList<>(); - for (int i = 0; i < reader.getRecordBlocks().size(); i++) { - reader.loadRecordBatch(reader.getRecordBlocks().get(i)); - VectorSchemaRoot root = reader.getVectorSchemaRoot(); - IntVector idVector = (IntVector) root.getVector("id"); - - // Extract ID values from this batch - for (int j = 0; j < root.getRowCount(); j++) { - if (!idVector.isNull(j)) { - idValues.add(idVector.get(j)); - } - } - } - - Collections.sort(idValues); - assertEquals(3, idValues.size(), "Should have exactly 3 rows"); - assertEquals( - Arrays.asList(2, 13, 14), idValues, "ID values should be [2, 13, 14] after updates"); - } - } - } finally { - DropTableResponse dropResponse = dropTableHelper(updateTableName); - } - } - - @Test - public void testDeleteFromTable() throws IOException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("\n=== Test: Delete From Table ==="); - String deleteTableName = "test_delete_table_" + System.currentTimeMillis(); - - try { - // Step 1: Create table with 3 rows - System.out.println("\n--- Step 1: Creating table with 3 rows ---"); - CreateTableResponse createResponse = createTableHelper(deleteTableName, 3); - assertNotNull(createResponse, "Create table response should not be null"); - - System.out.println("\n--- Step 2: Deleting row with id=1 ---"); - DeleteFromTableRequest deleteRequest = new DeleteFromTableRequest(); - deleteRequest.setName(deleteTableName); - deleteRequest.setPredicate("id = 1"); - - DeleteFromTableResponse deleteResponse = namespace.deleteFromTable(deleteRequest); - // Note that server didn't send back valid deletedRows info - assertEquals(2, deleteResponse.getVersion(), "Version should increment"); - - CountRowsRequest countRequest = new CountRowsRequest(); - countRequest.setName(deleteTableName); - Long countResponse = namespace.countRows(countRequest); - assertEquals(2, countResponse.longValue()); - - System.out.println("\n--- Step 5: Deleting with complex predicate (id > 2) ---"); - DeleteFromTableRequest complexDeleteRequest = new DeleteFromTableRequest(); - complexDeleteRequest.setName(deleteTableName); - complexDeleteRequest.setPredicate("id > 2"); - - DeleteFromTableResponse complexDeleteResponse = - namespace.deleteFromTable(complexDeleteRequest); - assertEquals(3, complexDeleteResponse.getVersion(), "Version should increment"); - countResponse = namespace.countRows(countRequest); - assertEquals(1, countResponse.longValue()); - } finally { - DropTableResponse dropResponse = dropTableHelper(deleteTableName); - } - } - - @Test - public void testMergeInsert() throws IOException { - assumeTrue( - DATABASE != null && API_KEY != null, - "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); - - System.out.println("\n=== Test: Merge Insert (Upsert) ==="); - String mergeTableName = "test_merge_table_" + System.currentTimeMillis(); - - try { - // Step 1: Create table with 3 rows - System.out.println("\n--- Step 1: Creating table with 3 rows ---"); - CreateTableResponse createResponse = createTableHelper(mergeTableName, 3); - assertNotNull(createResponse, "Create table response should not be null"); - - // Verify initial data - CountRowsRequest countRequest = new CountRowsRequest(); - countRequest.setName(mergeTableName); - Long initialCount = namespace.countRows(countRequest); - assertEquals(3, initialCount.longValue(), "Initial row count should be 3"); - - // Step 2: Merge insert with some matching and some new rows - System.out.println("\n--- Step 2: Merge insert with mixed rows ---"); - // Create batch with rows: id=2,3,4 (2,3 exist, 4 is new) - String[] names = {"Bob Updated", "Charlie Updated", "David"}; - Float4Vector[] vectors = new Float4Vector[3]; - - Schema schema = - new Schema( - Arrays.asList( - new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null), - new Field("name", FieldType.nullable(new ArrowType.Utf8()), null), - new Field( - "embedding", - FieldType.nullable(new ArrowType.FixedSizeList(128)), - Arrays.asList( - new Field( - "item", - FieldType.nullable( - new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), - null))))); - - try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { - IntVector idVector = (IntVector) root.getVector("id"); - VarCharVector nameVector = (VarCharVector) root.getVector("name"); - FixedSizeListVector vectorVector = (FixedSizeListVector) root.getVector("embedding"); - - root.setRowCount(3); - - // Set data for merge - int[] ids = {2, 3, 4}; - for (int i = 0; i < 3; i++) { - idVector.setSafe(i, ids[i]); - nameVector.setSafe(i, names[i].getBytes(StandardCharsets.UTF_8)); - } - - // Populate vector field - Float4Vector dataVector = (Float4Vector) vectorVector.getDataVector(); - vectorVector.allocateNew(); - - for (int row = 0; row < 3; row++) { - vectorVector.setNotNull(row); - for (int dim = 0; dim < 128; dim++) { - int index = row * 128 + dim; - dataVector.setSafe(index, (float) ((row + 2) * 10.0 + dim * 0.1)); - } - } - - idVector.setValueCount(3); - nameVector.setValueCount(3); - dataVector.setValueCount(3 * 128); - vectorVector.setValueCount(3); - - // Serialize to Arrow IPC - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try (ArrowStreamWriter writer = - new ArrowStreamWriter(root, null, Channels.newChannel(out))) { - writer.start(); - writer.writeBatch(); - writer.end(); - } - - byte[] arrowIpcData = out.toByteArray(); - - // Perform merge insert - MergeInsertTableRequest mergeRequest = new MergeInsertTableRequest(); - mergeRequest.setName(mergeTableName); - - MergeInsertTableResponse mergeResponse = - namespace.mergeInsertTable( - mergeRequest, - arrowIpcData, - "id", // match on id column - true, // when_matched_update_all - true // when_not_matched_insert_all - ); - - assertNotNull(mergeResponse, "Merge response should not be null"); - assertEquals( - 2, mergeResponse.getNumUpdatedRows().longValue(), "Should have updated 2 rows"); - assertEquals( - 1, mergeResponse.getNumInsertedRows().longValue(), "Should have inserted 1 row"); - System.out.println( - "✓ Merge insert completed: " - + mergeResponse.getNumUpdatedRows() - + " updated, " - + mergeResponse.getNumInsertedRows() - + " inserted"); - - // Verify final count - Long finalCount = namespace.countRows(countRequest); - assertEquals(4, finalCount.longValue(), "Final row count should be 4"); - - // Step 3: Verify the updates by querying the table - System.out.println("\n--- Step 3: Verifying merged data ---"); - QueryRequest queryRequest = new QueryRequest(); - queryRequest.setName(mergeTableName); - queryRequest.setK(4); - // Don't set columns or vector for simple queries - - byte[] queryResult = namespace.queryTable(queryRequest); - assertNotNull(queryResult, "Query result should not be null"); - - // Read and verify the results - try (BufferAllocator verifyAllocator = new RootAllocator()) { - ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(queryResult); - - try (ArrowFileReader reader = new ArrowFileReader(channel, verifyAllocator)) { - Map idToName = new HashMap<>(); - - for (int i = 0; i < reader.getRecordBlocks().size(); i++) { - reader.loadRecordBatch(reader.getRecordBlocks().get(i)); - VectorSchemaRoot resultRoot = reader.getVectorSchemaRoot(); - IntVector resultIdVector = (IntVector) resultRoot.getVector("id"); - VarCharVector resultNameVector = (VarCharVector) resultRoot.getVector("name"); - - for (int j = 0; j < resultRoot.getRowCount(); j++) { - if (!resultIdVector.isNull(j)) { - int id = resultIdVector.get(j); - String name = new String(resultNameVector.get(j), StandardCharsets.UTF_8); - idToName.put(id, name); - } - } - } - - // Verify the merged data - assertEquals(4, idToName.size(), "Should have 4 rows total"); - assertEquals("Alice", idToName.get(1), "ID 1 should remain unchanged"); - assertEquals("Bob Updated", idToName.get(2), "ID 2 should be updated"); - assertEquals("Charlie Updated", idToName.get(3), "ID 3 should be updated"); - assertEquals("David", idToName.get(4), "ID 4 should be new"); - System.out.println("✓ Merge insert data verified successfully"); - } - } - } - } finally { - DropTableResponse dropResponse = dropTableHelper(mergeTableName); - } - } - - /** SeekableByteChannel implementation for reading Arrow file format from byte array */ - private static class ByteArraySeekableByteChannel implements SeekableByteChannel { - private final byte[] data; - private long position = 0; - private boolean isOpen = true; - - public ByteArraySeekableByteChannel(byte[] data) { - this.data = data; - } - - @Override - public long position() throws IOException { - return position; - } - - @Override - public SeekableByteChannel position(long newPosition) throws IOException { - if (newPosition < 0 || newPosition > data.length) { - throw new IOException("Invalid position: " + newPosition); - } - position = newPosition; - return this; - } - - @Override - public long size() throws IOException { - return data.length; - } - - @Override - public int read(ByteBuffer dst) throws IOException { - if (!isOpen) { - throw new IOException("Channel is closed"); - } - int remaining = dst.remaining(); - int available = (int) (data.length - position); - if (available <= 0) { - return -1; - } - int toRead = Math.min(remaining, available); - dst.put(data, (int) position, toRead); - position += toRead; - return toRead; - } - - @Override - public int write(ByteBuffer src) throws IOException { - throw new UnsupportedOperationException("Read-only channel"); - } - - @Override - public SeekableByteChannel truncate(long size) throws IOException { - throw new UnsupportedOperationException("Read-only channel"); - } - - @Override - public boolean isOpen() { - return isOpen; - } - - @Override - public void close() throws IOException { - isOpen = false; - } - } -} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/BaseNamespaceTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/BaseNamespaceTest.java new file mode 100644 index 000000000..98925bd6f --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/BaseNamespaceTest.java @@ -0,0 +1,124 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test; + +import com.lancedb.lance.namespace.LanceRestNamespace; +import com.lancedb.lance.namespace.client.apache.ApiClient; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Base test class for Lance Namespace tests. Provides common setup and configuration for all test + * classes. + */ +public abstract class BaseNamespaceTest { + + // Configuration from environment variables + protected static String DATABASE; + protected static String API_KEY; + protected static String HOST_OVERRIDE; + protected static String REGION; + + protected LanceRestNamespace namespace; + protected BufferAllocator allocator; + + @BeforeAll + public static void setUpClass() { + // Get configuration from environment variables + DATABASE = System.getenv("LANCEDB_DB"); + API_KEY = System.getenv("LANCEDB_API_KEY"); + HOST_OVERRIDE = System.getenv("LANCEDB_HOST_OVERRIDE"); + REGION = System.getenv("LANCEDB_REGION"); + + // Default values if not set + if (REGION == null) { + REGION = "us-east-1"; + } + + // TODO(claude) only API_KEY is required. DATABASE can be a random database name + if (DATABASE != null && API_KEY != null) { + System.out.println("Using configuration:"); + System.out.println(" Database: " + DATABASE); + System.out.println(" Region: " + REGION); + System.out.println(" Host Override: " + (HOST_OVERRIDE != null ? HOST_OVERRIDE : "none")); + } + } + + @BeforeEach + public void setUp() { + // Only initialize if required environment variables are set + if (DATABASE != null && API_KEY != null) { + namespace = initializeClient(); + allocator = new RootAllocator(); + } + } + + @AfterEach + public void tearDown() { + if (allocator != null) { + allocator.close(); + } + } + + /** + * Skip test if environment variables are not set. Call this at the beginning of each test method. + */ + protected void skipIfNotConfigured() { + assumeTrue( + DATABASE != null && API_KEY != null, + "Skipping test: LANCEDB_DB and LANCEDB_API_KEY environment variables must be set"); + } + + /** + * Initialize the Lance REST client. + * + * @return Configured LanceRestNamespace instance + */ + private LanceRestNamespace initializeClient() { + Map config = new HashMap<>(); + config.put("headers.x-lancedb-database", DATABASE); + config.put("headers.x-api-key", API_KEY); + + if (HOST_OVERRIDE != null) { + config.put("host_override", HOST_OVERRIDE); + } + if (REGION != null) { + config.put("region", REGION); + } + + ApiClient apiClient = new ApiClient(); + + // Set base URL based on configuration + String baseUrl; + if (HOST_OVERRIDE != null) { + baseUrl = HOST_OVERRIDE; + } else { + baseUrl = String.format("https://%s.%s.api.lancedb.com", DATABASE, REGION); + } + apiClient.setBasePath(baseUrl); + + System.out.println("Initialized client with base URL: " + baseUrl); + + return new LanceRestNamespace(apiClient, config); + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java new file mode 100644 index 000000000..acdb51db0 --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java @@ -0,0 +1,296 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.index; + +import com.lancedb.lance.namespace.model.*; +import com.lancedb.lance.namespace.test.BaseNamespaceTest; +import com.lancedb.lance.namespace.test.utils.ArrowTestUtils; +import com.lancedb.lance.namespace.test.utils.TestUtils; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.*; + +/** Tests for index operations: create, list, stats. */ +public class IndexTest extends BaseNamespaceTest { + + @Test + public void testCreateVectorIndex() throws IOException, InterruptedException { + skipIfNotConfigured(); + + System.out.println("=== Test: Create Vector Index ==="); + String tableName = TestUtils.generateTableName("test_vector_index"); + + try { + // Step 1: Create table with 300 rows + System.out.println("\n--- Step 1: Creating table with 300 rows ---"); + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 300).build(); + CreateTableResponse createResponse = namespace.createTable(tableName, tableData); + assertNotNull(createResponse, "Create table response should not be null"); + + // Step 2: List indices before creating index (should be empty) + System.out.println("\n--- Step 2: Listing indices before index creation ---"); + IndexListRequest listRequestBefore = new IndexListRequest(); + listRequestBefore.setName(tableName); + IndexListResponse listResponseBefore = namespace.listIndices(listRequestBefore); + assertNotNull(listResponseBefore, "List indices response should not be null"); + assertNotNull(listResponseBefore.getIndexes(), "Indexes list should not be null"); + assertEquals( + 0, listResponseBefore.getIndexes().size(), "Should have no indices before creation"); + System.out.println("✓ Confirmed no indices exist before creation"); + + // Step 3: Create vector index + System.out.println("\n--- Step 3: Creating vector index ---"); + CreateIndexRequest indexRequest = new CreateIndexRequest(); + indexRequest.setName(tableName); + indexRequest.setColumn("embedding"); + indexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.IVF_PQ); + indexRequest.setMetricType(CreateIndexRequest.MetricTypeEnum.L2); + + CreateIndexResponse indexResponse = namespace.createIndex(indexRequest); + assertNotNull(indexResponse, "Create index response should not be null"); + System.out.println("✓ Index creation request submitted successfully"); + + // Step 4: Wait for index creation completion + System.out.println("\n--- Step 4: Waiting for index creation to complete ---"); + boolean indexFound = TestUtils.waitForIndex(namespace, tableName, "embedding_idx", 60); + assertTrue(indexFound, "Index should be found after creation"); + System.out.println("✓ Index created successfully"); + + // Step 5: Get index stats + System.out.println("\n--- Step 5: Getting index stats for embedding_idx ---"); + IndexStatsRequest statsRequest = new IndexStatsRequest(); + statsRequest.setName(tableName); + + IndexStatsResponse statsResponse = namespace.getIndexStats(statsRequest, "embedding_idx"); + assertNotNull(statsResponse, "Index stats response should not be null"); + assertEquals("IVF_PQ", statsResponse.getIndexType(), "Index type should be IVF_PQ"); + assertEquals("l2", statsResponse.getDistanceType(), "Distance type should be l2"); + + // Wait for index to be fully built + boolean indexComplete = + TestUtils.waitForIndexComplete(namespace, tableName, "embedding_idx", 30); + assertTrue(indexComplete, "Index should be fully built"); + + // Verify final stats + IndexStatsResponse finalStats = namespace.getIndexStats(statsRequest, "embedding_idx"); + assertEquals(300, finalStats.getNumIndexedRows().intValue(), "Should have 300 indexed rows"); + assertEquals(0, finalStats.getNumUnindexedRows().intValue(), "Should have 0 unindexed rows"); + + System.out.println("✓ Index stats verified:"); + System.out.println(" - Type: " + finalStats.getIndexType()); + System.out.println(" - Distance: " + finalStats.getDistanceType()); + System.out.println(" - Indexed rows: " + finalStats.getNumIndexedRows()); + System.out.println(" - Unindexed rows: " + finalStats.getNumUnindexedRows()); + + // Verify index appears in list + IndexListRequest listRequestAfter = new IndexListRequest(); + listRequestAfter.setName(tableName); + IndexListResponse listResponseAfter = namespace.listIndices(listRequestAfter); + assertEquals(1, listResponseAfter.getIndexes().size(), "Should have exactly one index"); + assertEquals( + "embedding_idx", + listResponseAfter.getIndexes().get(0).getIndexName(), + "Index name should be embedding_idx"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testCreateScalarIndex() throws IOException, InterruptedException { + skipIfNotConfigured(); + + System.out.println("=== Test: Create Scalar Index ==="); + String tableName = TestUtils.generateTableName("test_scalar_index"); + + try { + // Step 1: Create table with 300 rows + System.out.println("\n--- Step 1: Creating table with 300 rows ---"); + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 300).build(); + CreateTableResponse createResponse = namespace.createTable(tableName, tableData); + assertNotNull(createResponse, "Create table response should not be null"); + + // Step 2: List indices before creating index + System.out.println("\n--- Step 2: Listing indices before index creation ---"); + IndexListRequest listRequestBefore = new IndexListRequest(); + listRequestBefore.setName(tableName); + IndexListResponse listResponseBefore = namespace.listIndices(listRequestBefore); + assertEquals( + 0, listResponseBefore.getIndexes().size(), "Should have no indices before creation"); + + // Step 3: Create scalar index on name column + System.out.println("\n--- Step 3: Creating scalar index ---"); + CreateIndexRequest scalarIndexRequest = new CreateIndexRequest(); + scalarIndexRequest.setName(tableName); + scalarIndexRequest.setColumn("name"); + scalarIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.BITMAP); + + CreateIndexResponse scalarIndexResponse = namespace.createScalarIndex(scalarIndexRequest); + assertNotNull(scalarIndexResponse, "Create scalar index response should not be null"); + System.out.println("✓ Scalar index creation request submitted"); + + // Step 4: Wait for index creation completion + System.out.println("\n--- Step 4: Waiting for scalar index creation to complete ---"); + boolean indexFound = TestUtils.waitForIndex(namespace, tableName, "name_idx", 60); + assertTrue(indexFound, "Scalar index should be found after creation"); + + // Step 5: Verify scalar index stats + System.out.println("\n--- Step 5: Getting scalar index stats ---"); + boolean indexComplete = TestUtils.waitForIndexComplete(namespace, tableName, "name_idx", 30); + assertTrue(indexComplete, "Scalar index should be fully built"); + + IndexStatsRequest statsRequest = new IndexStatsRequest(); + statsRequest.setName(tableName); + IndexStatsResponse statsResponse = namespace.getIndexStats(statsRequest, "name_idx"); + + assertNotNull(statsResponse, "Index stats response should not be null"); + assertEquals("BITMAP", statsResponse.getIndexType(), "Index type should be BITMAP"); + assertEquals( + 300, statsResponse.getNumIndexedRows().intValue(), "Should have 300 indexed rows"); + assertEquals( + 0, statsResponse.getNumUnindexedRows().intValue(), "Should have 0 unindexed rows"); + + System.out.println("✓ Scalar index stats verified:"); + System.out.println(" - Type: " + statsResponse.getIndexType()); + System.out.println(" - Indexed rows: " + statsResponse.getNumIndexedRows()); + + // Verify index in list + IndexListRequest listRequestAfter = new IndexListRequest(); + listRequestAfter.setName(tableName); + IndexListResponse listResponseAfter = namespace.listIndices(listRequestAfter); + assertEquals(1, listResponseAfter.getIndexes().size(), "Should have exactly one index"); + assertTrue( + listResponseAfter.getIndexes().get(0).getColumns().contains("name"), + "Index should be on name column"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testMultipleIndices() throws IOException, InterruptedException { + skipIfNotConfigured(); + + System.out.println("=== Test: Multiple Indices ==="); + String tableName = TestUtils.generateTableName("test_multiple_indices"); + + try { + // Create table with text field + System.out.println("\n--- Creating table with multiple fields ---"); + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 100).build(); + namespace.createTable(tableName, tableData); + + // Create vector index + System.out.println("\n--- Creating vector index ---"); + CreateIndexRequest vectorIndexRequest = new CreateIndexRequest(); + vectorIndexRequest.setName(tableName); + vectorIndexRequest.setColumn("embedding"); + vectorIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.IVF_PQ); + vectorIndexRequest.setMetricType(CreateIndexRequest.MetricTypeEnum.COSINE); + namespace.createIndex(vectorIndexRequest); + + // Create scalar index + System.out.println("\n--- Creating scalar index ---"); + CreateIndexRequest scalarIndexRequest = new CreateIndexRequest(); + scalarIndexRequest.setName(tableName); + scalarIndexRequest.setColumn("id"); + scalarIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.BTREE); + namespace.createScalarIndex(scalarIndexRequest); + + // Create FTS index + System.out.println("\n--- Creating FTS index ---"); + CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); + ftsIndexRequest.setName(tableName); + ftsIndexRequest.setColumn("text"); + ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); + namespace.createIndex(ftsIndexRequest); + + // Wait for all indices + System.out.println("\n--- Waiting for all indices to be created ---"); + Thread.sleep(5000); // Give some time for all indices to appear + + // List all indices + IndexListRequest listRequest = new IndexListRequest(); + listRequest.setName(tableName); + IndexListResponse listResponse = namespace.listIndices(listRequest); + + assertNotNull(listResponse.getIndexes(), "Indexes list should not be null"); + assertTrue(listResponse.getIndexes().size() >= 3, "Should have at least 3 indices"); + + System.out.println("✓ Created " + listResponse.getIndexes().size() + " indices:"); + for (IndexListItemResponse index : listResponse.getIndexes()) { + System.out.println(" - " + index.getIndexName() + " on columns: " + index.getColumns()); + } + + // Verify each index type + boolean hasVectorIndex = + listResponse.getIndexes().stream() + .anyMatch(idx -> idx.getIndexName().equals("embedding_idx")); + boolean hasScalarIndex = + listResponse.getIndexes().stream().anyMatch(idx -> idx.getIndexName().equals("id_idx")); + boolean hasFtsIndex = + listResponse.getIndexes().stream().anyMatch(idx -> idx.getIndexName().equals("text_idx")); + + assertTrue(hasVectorIndex, "Should have vector index"); + assertTrue(hasScalarIndex, "Should have scalar index"); + assertTrue(hasFtsIndex, "Should have FTS index"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testIndexWithDifferentMetrics() throws IOException, InterruptedException { + skipIfNotConfigured(); + + System.out.println("=== Test: Index with Different Metrics ==="); + String tableName = TestUtils.generateTableName("test_index_metrics"); + + try { + // Create table + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 200).build(); + namespace.createTable(tableName, tableData); + + // Test COSINE metric + System.out.println("\n--- Creating index with COSINE metric ---"); + CreateIndexRequest cosineRequest = new CreateIndexRequest(); + cosineRequest.setName(tableName); + cosineRequest.setColumn("embedding"); + cosineRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.IVF_PQ); + cosineRequest.setMetricType(CreateIndexRequest.MetricTypeEnum.COSINE); + + CreateIndexResponse cosineResponse = namespace.createIndex(cosineRequest); + assertNotNull(cosineResponse, "Create index response should not be null"); + + // Wait and verify + TestUtils.waitForIndexComplete(namespace, tableName, "embedding_idx", 30); + + IndexStatsRequest statsRequest = new IndexStatsRequest(); + statsRequest.setName(tableName); + IndexStatsResponse statsResponse = namespace.getIndexStats(statsRequest, "embedding_idx"); + + assertEquals("cosine", statsResponse.getDistanceType(), "Distance type should be cosine"); + System.out.println("✓ Created index with COSINE metric"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java new file mode 100644 index 000000000..ba4162a2c --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java @@ -0,0 +1,450 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.query; + +import com.lancedb.lance.namespace.model.*; +import com.lancedb.lance.namespace.test.BaseNamespaceTest; +import com.lancedb.lance.namespace.test.utils.ArrowTestUtils; +import com.lancedb.lance.namespace.test.utils.TestUtils; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** Tests for full-text search functionality. */ +public class FullTextSearchTest extends BaseNamespaceTest { + + @Test + public void testSimpleFullTextSearch() throws IOException, InterruptedException { + skipIfNotConfigured(); + + System.out.println("=== Test: Simple Full-Text Search ==="); + String tableName = TestUtils.generateTableName("test_fts"); + + try { + // Create table with deterministic text data + System.out.println("\n--- Creating table with text data ---"); + byte[] tableData = + new ArrowTestUtils.TableDataBuilder(allocator) + .withSchema(ArrowTestUtils.createSchemaWithText(128)) + .withText(1, "This is an important document about search functionality") + .withText(2, "Another document containing critical information") + .withText(3, "Sample text with important data and keywords") + .withText(4, "Regular content without special keywords") + .withText(5, "Document with search terms and important notes") + .addRows(1, 5) + .build(); + + CreateTableResponse createResponse = namespace.createTable(tableName, tableData); + assertNotNull(createResponse, "Create response should not be null"); + + // Create FTS index + System.out.println("\n--- Creating FTS index ---"); + CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); + ftsIndexRequest.setName(tableName); + ftsIndexRequest.setColumn("text"); + ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); + + CreateIndexResponse ftsIndexResponse = namespace.createIndex(ftsIndexRequest); + assertNotNull(ftsIndexResponse, "FTS index response should not be null"); + + TestUtils.waitForIndexComplete(namespace, tableName, "text_idx", 30); + + // Test 1: Simple string FTS query for "document" + System.out.println("\n--- Test 1: Simple string FTS query for 'document' ---"); + System.out.println("Expected to find rows 1, 2, 5 (contain 'document')"); + QueryRequest stringFtsQuery = new QueryRequest(); + stringFtsQuery.setName(tableName); + stringFtsQuery.setK(10); + stringFtsQuery.setColumns(Arrays.asList("id", "name", "text", "category")); + + StringFtsQuery simpleFts = new StringFtsQuery(); + simpleFts.setQuery("document"); + QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); + fullTextQuery.setStringQuery(simpleFts); + stringFtsQuery.setFullTextQuery(fullTextQuery); + + byte[] ftsResult = namespace.queryTable(stringFtsQuery); + assertNotNull(ftsResult, "FTS query result should not be null"); + + int rowCount = ArrowTestUtils.countRows(ftsResult, allocator); + assertEquals(3, rowCount, "Should find exactly 3 rows containing 'document'"); + + List foundIds = + ArrowTestUtils.extractColumn(ftsResult, allocator, "id", Integer.class); + assertTrue(foundIds.contains(1), "Should find row 1"); + assertTrue(foundIds.contains(2), "Should find row 2"); + assertTrue(foundIds.contains(5), "Should find row 5"); + + System.out.println("✓ FTS query returned " + rowCount + " results with IDs: " + foundIds); + + // Test 2: FTS query for "important" + System.out.println("\n--- Test 2: FTS query for 'important' ---"); + System.out.println("Expected to find rows 1, 3, 5 (contain 'important')"); + QueryRequest columnFtsQuery = new QueryRequest(); + columnFtsQuery.setName(tableName); + columnFtsQuery.setK(5); + columnFtsQuery.setColumns(Arrays.asList("id", "name", "text")); + + StringFtsQuery columnFts = new StringFtsQuery(); + columnFts.setQuery("important"); + columnFts.setColumns(Arrays.asList("text")); // Search only in text column + QueryRequestFullTextQuery columnFullTextQuery = new QueryRequestFullTextQuery(); + columnFullTextQuery.setStringQuery(columnFts); + columnFtsQuery.setFullTextQuery(columnFullTextQuery); + + byte[] columnFtsResult = namespace.queryTable(columnFtsQuery); + assertNotNull(columnFtsResult, "Column FTS query result should not be null"); + + int importantCount = ArrowTestUtils.countRows(columnFtsResult, allocator); + assertEquals(3, importantCount, "Should find exactly 3 rows containing 'important'"); + + List importantIds = + ArrowTestUtils.extractColumn(columnFtsResult, allocator, "id", Integer.class); + assertTrue(importantIds.contains(1), "Should find row 1"); + assertTrue(importantIds.contains(3), "Should find row 3"); + assertTrue(importantIds.contains(5), "Should find row 5"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testFullTextSearchWithFilter() throws IOException, InterruptedException { + skipIfNotConfigured(); + + System.out.println("=== Test: Full-Text Search with Filter ==="); + String tableName = TestUtils.generateTableName("test_fts_filter"); + + try { + // Create table with deterministic text data + System.out.println("\n--- Creating table with text data ---"); + ArrowTestUtils.TableDataBuilder builder = + new ArrowTestUtils.TableDataBuilder(allocator) + .withSchema(ArrowTestUtils.createSchemaWithText(128)); + + // Add rows with specific text patterns + for (int i = 1; i <= 30; i++) { + if (i % 3 == 0) { + builder.withText(i, "This is document " + i + " with important information"); + } else if (i % 5 == 0) { + builder.withText(i, "Document " + i + " contains critical data"); + } else { + builder.withText(i, "Regular text for row " + i); + } + } + + byte[] tableData = builder.addRows(1, 30).build(); + + namespace.createTable(tableName, tableData); + + // Create FTS index + CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); + ftsIndexRequest.setName(tableName); + ftsIndexRequest.setColumn("text"); + ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); + namespace.createIndex(ftsIndexRequest); + TestUtils.waitForIndexComplete(namespace, tableName, "text_idx", 30); + + // FTS with prefilter + System.out.println("\n--- Testing FTS with prefilter (id < 25) ---"); + QueryRequest ftsPrefilterQuery = new QueryRequest(); + ftsPrefilterQuery.setName(tableName); + ftsPrefilterQuery.setK(20); // Set high to get all matches + ftsPrefilterQuery.setPrefilter(true); + ftsPrefilterQuery.setFilter("id < 25"); + ftsPrefilterQuery.setColumns(Arrays.asList("id", "text")); + + StringFtsQuery fts = new StringFtsQuery(); + fts.setQuery("document"); + QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); + fullTextQuery.setStringQuery(fts); + ftsPrefilterQuery.setFullTextQuery(fullTextQuery); + + byte[] ftsPrefilterResult = namespace.queryTable(ftsPrefilterQuery); + assertNotNull(ftsPrefilterResult, "FTS prefilter query result should not be null"); + + List ids = + ArrowTestUtils.extractColumn(ftsPrefilterResult, allocator, "id", Integer.class); + assertTrue( + ids.stream().allMatch(id -> id < 25), "All IDs should be less than 25 with prefilter"); + + // Expected document rows under 25: multiples of 3 or 5 + List expectedIds = Arrays.asList(3, 5, 6, 9, 10, 12, 15, 18, 20, 21, 24); + assertEquals( + expectedIds.size(), + ids.size(), + "Should find exactly " + expectedIds.size() + " documents under ID 25"); + + for (Integer expectedId : expectedIds) { + assertTrue(ids.contains(expectedId), "Should find row " + expectedId); + } + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testStructuredFullTextSearch() throws IOException, InterruptedException { + skipIfNotConfigured(); + + System.out.println("=== Test: Structured Full-Text Search ==="); + String tableName = TestUtils.generateTableName("test_structured_fts"); + + try { + // Create table with specific text patterns for boolean query testing + System.out.println("\n--- Creating table with text data for structured queries ---"); + ArrowTestUtils.TableDataBuilder builder = + new ArrowTestUtils.TableDataBuilder(allocator) + .withSchema(ArrowTestUtils.createSchemaWithText(128)); + + // Create specific patterns: + // Rows 1-5: contain both "important" and "document" + builder + .withText(1, "This is an important document about testing") + .withText(2, "Another important document with data") + .withText(3, "Important technical document here") + .withText(4, "Document containing important information") + .withText(5, "Very important document to read"); + + // Rows 6-8: contain only "important" + builder + .withText(6, "Important notice without d-word") + .withText(7, "This is important text") + .withText(8, "Important data here"); + + // Rows 9-11: contain only "document" + builder + .withText(9, "Simple document text") + .withText(10, "Another document here") + .withText(11, "Document without i-word"); + + // Rows 12-15: contain neither + builder + .withText(12, "Regular text content") + .withText(13, "Simple data entry") + .withText(14, "Basic information") + .withText(15, "Normal text here"); + + byte[] tableData = builder.addRows(1, 15).build(); + + System.out.println("Created table with 15 rows:"); + System.out.println(" - Rows 1-5: contain both 'important' AND 'document'"); + System.out.println(" - Rows 6-8: contain only 'important'"); + System.out.println(" - Rows 9-11: contain only 'document'"); + System.out.println(" - Rows 12-15: contain neither"); + + namespace.createTable(tableName, tableData); + + // Create FTS index + CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); + ftsIndexRequest.setName(tableName); + ftsIndexRequest.setColumn("text"); + ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); + namespace.createIndex(ftsIndexRequest); + TestUtils.waitForIndexComplete(namespace, tableName, "text_idx", 30); + + // Test Boolean Query + System.out.println("\n--- Testing Boolean Query ---"); + QueryRequest booleanQuery = new QueryRequest(); + booleanQuery.setName(tableName); + booleanQuery.setK(10); + booleanQuery.setColumns(Arrays.asList("id", "text")); + + // Create boolean query: MUST contain "important" AND SHOULD contain "document" + QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); + StructuredFtsQuery structured = new StructuredFtsQuery(); + FtsQuery ftsQuery = new FtsQuery(); + + BooleanQuery boolQuery = new BooleanQuery(); + + // Must clause + FtsQuery mustQuery = new FtsQuery(); + MatchQuery mustMatch = new MatchQuery(); + mustMatch.setTerms("important"); + mustMatch.setColumn("text"); + mustQuery.setMatch(mustMatch); + boolQuery.setMust(Arrays.asList(mustQuery)); + + // Should clause + FtsQuery shouldQuery = new FtsQuery(); + MatchQuery shouldMatch = new MatchQuery(); + shouldMatch.setTerms("document"); + shouldQuery.setMatch(shouldMatch); + boolQuery.setShould(Arrays.asList(shouldQuery)); + + ftsQuery.setBoolean(boolQuery); + structured.setQuery(ftsQuery); + fullTextQuery.setStructuredQuery(structured); + booleanQuery.setFullTextQuery(fullTextQuery); + + byte[] boolResult = namespace.queryTable(booleanQuery); + assertNotNull(boolResult, "Boolean query result should not be null"); + + int boolRowCount = ArrowTestUtils.countRows(boolResult, allocator); + assertEquals(8, boolRowCount, "Should find exactly 8 rows with 'important'"); + + List boolIds = + ArrowTestUtils.extractColumn(boolResult, allocator, "id", Integer.class); + for (int i = 1; i <= 8; i++) { + assertTrue(boolIds.contains(i), "Should find row " + i); + } + + // Test Phrase Query + System.out.println("\n--- Testing Phrase Query ---"); + QueryRequest phraseQuery = new QueryRequest(); + phraseQuery.setName(tableName); + phraseQuery.setK(10); + phraseQuery.setColumns(Arrays.asList("id", "text")); + + QueryRequestFullTextQuery phraseFullTextQuery = new QueryRequestFullTextQuery(); + StructuredFtsQuery phraseStructured = new StructuredFtsQuery(); + FtsQuery phraseFtsQuery = new FtsQuery(); + + PhraseQuery phrase = new PhraseQuery(); + phrase.setTerms("important document"); + phrase.setColumn("text"); + phrase.setSlop(1); // Allow 1 word between terms + phraseFtsQuery.setPhrase(phrase); + + phraseStructured.setQuery(phraseFtsQuery); + phraseFullTextQuery.setStructuredQuery(phraseStructured); + phraseQuery.setFullTextQuery(phraseFullTextQuery); + + byte[] phraseResult = namespace.queryTable(phraseQuery); + assertNotNull(phraseResult, "Phrase query result should not be null"); + + int phraseRowCount = ArrowTestUtils.countRows(phraseResult, allocator); + assertEquals( + 5, phraseRowCount, "Should find exactly 5 rows with 'important' near 'document'"); + + List phraseIds = + ArrowTestUtils.extractColumn(phraseResult, allocator, "id", Integer.class); + for (int i = 1; i <= 5; i++) { + assertTrue(phraseIds.contains(i), "Should find row " + i); + } + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testHybridSearch() throws IOException, InterruptedException { + skipIfNotConfigured(); + + System.out.println("=== Test: Hybrid Search (Vector + Full-Text) ==="); + String tableName = TestUtils.generateTableName("test_hybrid"); + + try { + // Create table with text and vector data + System.out.println("\n--- Creating table with text and vector data ---"); + ArrowTestUtils.TableDataBuilder builder = + new ArrowTestUtils.TableDataBuilder(allocator) + .withSchema(ArrowTestUtils.createSchemaWithText(128)); + + // Add 20 rows with specific patterns + for (int i = 1; i <= 20; i++) { + if (i <= 5) { + builder.withText(i, "Important document about vector search"); + } else if (i <= 10) { + builder.withText(i, "Regular document with some content"); + } else if (i <= 15) { + builder.withText(i, "Basic text without keywords"); + } else { + builder.withText(i, "Another document for testing"); + } + } + + byte[] tableData = builder.addRows(1, 20).build(); + namespace.createTable(tableName, tableData); + + // Create FTS index + CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); + ftsIndexRequest.setName(tableName); + ftsIndexRequest.setColumn("text"); + ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); + namespace.createIndex(ftsIndexRequest); + TestUtils.waitForIndexComplete(namespace, tableName, "text_idx", 30); + + // Hybrid search: vector + text + System.out.println("\n--- Testing hybrid search ---"); + QueryRequest hybridQuery = new QueryRequest(); + hybridQuery.setName(tableName); + hybridQuery.setColumns(Arrays.asList("id", "text")); + + // Add vector search - search for vector with all 10s + List queryVector = new ArrayList<>(); + for (int i = 0; i < 128; i++) { + queryVector.add(10.0f); + } + hybridQuery.setVector(queryVector); + hybridQuery.setK(10); + + // Add full-text search + StringFtsQuery ftsQuery = new StringFtsQuery(); + ftsQuery.setQuery("document"); + QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); + fullTextQuery.setStringQuery(ftsQuery); + hybridQuery.setFullTextQuery(fullTextQuery); + + byte[] hybridResult = namespace.queryTable(hybridQuery); + assertNotNull(hybridResult, "Hybrid search result should not be null"); + + int hybridRowCount = ArrowTestUtils.countRows(hybridResult, allocator); + assertEquals( + 10, hybridRowCount, "Should find exactly 10 rows (1-10, 16-20 contain 'document')"); + + List hybridIds = + ArrowTestUtils.extractColumn(hybridResult, allocator, "id", Integer.class); + + // Verify all document-containing rows are found + for (int i = 1; i <= 10; i++) { + assertTrue(hybridIds.contains(i), "Should find row " + i); + } + for (int i = 16; i <= 20; i++) { + assertTrue(hybridIds.contains(i), "Should find row " + i); + } + // Hybrid search with filter + System.out.println("\n--- Testing hybrid search with filter (id <= 10) ---"); + QueryRequest hybridFilterQuery = new QueryRequest(); + hybridFilterQuery.setName(tableName); + hybridFilterQuery.setVector(queryVector); + hybridFilterQuery.setK(20); + hybridFilterQuery.setFilter("id <= 10"); + hybridFilterQuery.setFullTextQuery(fullTextQuery); + hybridFilterQuery.setColumns(Arrays.asList("id", "text")); + + byte[] hybridFilterResult = namespace.queryTable(hybridFilterQuery); + assertNotNull(hybridFilterResult, "Hybrid search with filter result should not be null"); + + List ids = + ArrowTestUtils.extractColumn(hybridFilterResult, allocator, "id", Integer.class); + assertEquals(10, ids.size(), "Should find exactly 10 rows (filter limits to id <= 10)"); + assertTrue(ids.stream().allMatch(id -> id <= 10), "All IDs should be <= 10"); + + for (int i = 1; i <= 10; i++) { + assertTrue(ids.contains(i), "Should find row " + i); + } + } finally { + TestUtils.dropTable(namespace, tableName); + } + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/QueryTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/QueryTest.java new file mode 100644 index 000000000..08cc1ddaf --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/QueryTest.java @@ -0,0 +1,267 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.query; + +import com.lancedb.lance.namespace.model.*; +import com.lancedb.lance.namespace.test.BaseNamespaceTest; +import com.lancedb.lance.namespace.test.utils.ArrowTestUtils; +import com.lancedb.lance.namespace.test.utils.TestUtils; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** Tests for query operations including vector search, filters, and various query options. */ +public class QueryTest extends BaseNamespaceTest { + + @Test + public void testBasicVectorQuery() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Basic Vector Query ==="); + String tableName = TestUtils.generateTableName("test_vector_query"); + + try { + // Create table with 10 rows + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 10).build(); + namespace.createTable(tableName, tableData); + + // Create vector query + QueryRequest queryRequest = TestUtils.createVectorQuery(tableName, 5, 128); + + queryRequest.setK(5); + + byte[] queryResult = namespace.queryTable(queryRequest); + assertNotNull(queryResult, "Query result should not be null"); + + // Verify results + try (BufferAllocator verifyAllocator = new RootAllocator()) { + ArrowTestUtils.readArrowFile( + queryResult, + verifyAllocator, + root -> { + Schema resultSchema = root.getSchema(); + List resultColumns = new ArrayList<>(); + for (Field field : resultSchema.getFields()) { + resultColumns.add(field.getName()); + } + System.out.println("Result columns: " + resultColumns); + + // Verify columns + assertTrue(resultColumns.contains("id"), "Result should contain 'id' column"); + assertTrue(resultColumns.contains("name"), "Result should contain 'name' column"); + }); + + int totalRows = ArrowTestUtils.countRows(queryResult, verifyAllocator); + System.out.println("Query returned " + totalRows + " rows"); + assertTrue(totalRows == 5, "Query should return exactly 5 rows"); + } + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testQueryWithFilter() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Query with Filter ==="); + String tableName = TestUtils.generateTableName("test_query_filter"); + + try { + // Create table with 100 rows for better filter testing + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 100).build(); + namespace.createTable(tableName, tableData); + + // Test 1: Filter-only query (no vector) + System.out.println("\n--- Test 1: Filter-only query ---"); + QueryRequest filterQuery = new QueryRequest(); + filterQuery.setName(tableName); + filterQuery.setK(10); + filterQuery.setFilter("id > 50"); + filterQuery.setColumns(Arrays.asList("id", "name", "embedding")); + + byte[] filterResult = namespace.queryTable(filterQuery); + assertNotNull(filterResult, "Filter query result should not be null"); + + int rowCount = ArrowTestUtils.countRows(filterResult, allocator); + assertEquals(10, rowCount, "Filter query should return exactly 10 rows"); + System.out.println("✓ Filter-only query returned " + rowCount + " rows"); + + // Test 2: Vector query with filter + System.out.println("\n--- Test 2: Vector query with filter ---"); + QueryRequest vectorFilterQuery = TestUtils.createVectorQuery(tableName, 5, 128); + vectorFilterQuery.setFilter("id < 20"); + + byte[] vectorFilterResult = namespace.queryTable(vectorFilterQuery); + assertNotNull(vectorFilterResult, "Vector filter query result should not be null"); + + List ids = + ArrowTestUtils.extractColumn(vectorFilterResult, allocator, "id", Integer.class); + assertTrue(ids.stream().allMatch(id -> id < 20), "All IDs should be less than 20"); + System.out.println("✓ Vector query with filter returned IDs: " + ids); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testQueryWithPrefilter() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Query with Prefilter ==="); + String tableName = TestUtils.generateTableName("test_prefilter"); + + try { + // Create table + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 50).build(); + namespace.createTable(tableName, tableData); + + // Test prefilter = true + System.out.println("\n--- Testing prefilter = true ---"); + QueryRequest prefilterQuery = new QueryRequest(); + prefilterQuery.setName(tableName); + prefilterQuery.setK(5); + prefilterQuery.setPrefilter(true); + prefilterQuery.setFilter("id < 20"); + prefilterQuery.setColumns(Arrays.asList("id", "name")); + + byte[] prefilterResult = namespace.queryTable(prefilterQuery); + assertNotNull(prefilterResult, "Prefilter query result should not be null"); + + List ids = + ArrowTestUtils.extractColumn(prefilterResult, allocator, "id", Integer.class); + assertTrue( + ids.stream().allMatch(id -> id < 20), "All IDs should be less than 20 with prefilter"); + System.out.println("✓ Prefilter query returned " + ids.size() + " rows"); + + // Test prefilter = false (postfilter) + System.out.println("\n--- Testing prefilter = false (postfilter) ---"); + QueryRequest postfilterQuery = TestUtils.createVectorQuery(tableName, 10, 128); + postfilterQuery.setPrefilter(false); + postfilterQuery.setFilter("id % 2 = 0"); // Even IDs only + + byte[] postfilterResult = namespace.queryTable(postfilterQuery); + assertNotNull(postfilterResult, "Postfilter query result should not be null"); + + List evenIds = + ArrowTestUtils.extractColumn(postfilterResult, allocator, "id", Integer.class); + assertTrue( + evenIds.stream().allMatch(id -> id % 2 == 0), "All IDs should be even with postfilter"); + System.out.println("✓ Postfilter query returned " + evenIds.size() + " even IDs"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testQueryWithFastSearch() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Query with Fast Search ==="); + String tableName = TestUtils.generateTableName("test_fast_search"); + + try { + // Create table + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 100).build(); + namespace.createTable(tableName, tableData); + + // Test fast_search = true + System.out.println("\n--- Testing fast_search = true ---"); + QueryRequest fastSearchQuery = TestUtils.createVectorQuery(tableName, 10, 128); + fastSearchQuery.setFastSearch(true); + + byte[] fastSearchResult = namespace.queryTable(fastSearchQuery); + assertNotNull(fastSearchResult, "Fast search query result should not be null"); + + int fastSearchRows = ArrowTestUtils.countRows(fastSearchResult, allocator); + assertEquals(10, fastSearchRows, "Fast search should return requested k rows"); + System.out.println("✓ Fast search returned " + fastSearchRows + " rows"); + + // Test fast_search = false + System.out.println("\n--- Testing fast_search = false ---"); + QueryRequest noFastSearchQuery = TestUtils.createVectorQuery(tableName, 10, 128); + noFastSearchQuery.setFastSearch(false); + + byte[] noFastSearchResult = namespace.queryTable(noFastSearchQuery); + assertNotNull(noFastSearchResult, "No fast search query result should not be null"); + + int noFastSearchRows = ArrowTestUtils.countRows(noFastSearchResult, allocator); + assertEquals(10, noFastSearchRows, "No fast search should also return requested k rows"); + System.out.println("✓ No fast search returned " + noFastSearchRows + " rows"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testQueryWithColumnSelection() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Query with Column Selection ==="); + String tableName = TestUtils.generateTableName("test_column_selection"); + + try { + // Create table + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 20).build(); + namespace.createTable(tableName, tableData); + + // Query with specific columns + QueryRequest columnQuery = new QueryRequest(); + columnQuery.setName(tableName); + columnQuery.setK(5); + columnQuery.setColumns(Arrays.asList("id", "name")); // Don't include embedding + + byte[] result = namespace.queryTable(columnQuery); + assertNotNull(result, "Column selection query result should not be null"); + + // Verify only requested columns are returned + try (BufferAllocator verifyAllocator = new RootAllocator()) { + ArrowTestUtils.readArrowFile( + result, + verifyAllocator, + root -> { + Schema schema = root.getSchema(); + List fieldNames = new ArrayList<>(); + for (Field field : schema.getFields()) { + fieldNames.add(field.getName()); + } + + assertTrue(fieldNames.contains("id"), "Result should contain 'id' column"); + assertTrue(fieldNames.contains("name"), "Result should contain 'name' column"); + assertFalse( + fieldNames.contains("embedding"), "Result should NOT contain 'embedding' column"); + + System.out.println("✓ Query returned only requested columns: " + fieldNames); + }); + } + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableLifecycleTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableLifecycleTest.java new file mode 100644 index 000000000..3eb1a4832 --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableLifecycleTest.java @@ -0,0 +1,217 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.table; + +import com.lancedb.lance.namespace.LanceNamespaceException; +import com.lancedb.lance.namespace.model.*; +import com.lancedb.lance.namespace.test.BaseNamespaceTest; +import com.lancedb.lance.namespace.test.utils.ArrowTestUtils; +import com.lancedb.lance.namespace.test.utils.TestUtils; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +/** Tests for table lifecycle operations: create, describe, insert, drop. */ +public class TableLifecycleTest extends BaseNamespaceTest { + + @Test + public void testTableLifecycle() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Table Lifecycle ==="); + String tableName = TestUtils.generateTableName("test_lifecycle"); + + try { + // Create table with 3 rows + System.out.println("\n--- Creating table ---"); + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 3).build(); + + CreateTableResponse createResponse = namespace.createTable(tableName, tableData); + assertNotNull(createResponse, "Create response should not be null"); + System.out.println("✓ Table created successfully: " + tableName); + + // Test count rows + System.out.println("\n--- Testing count rows ---"); + long count = TestUtils.countRows(namespace, tableName); + assertEquals(3, count, "Row count should match expected number"); + System.out.println("✓ Count rows verified: " + count); + + // Test describe table + System.out.println("\n--- Testing describe table ---"); + DescribeTableRequest describeRequest = new DescribeTableRequest(); + describeRequest.setName(tableName); + + DescribeTableResponse describeResponse = namespace.describeTable(describeRequest); + assertNotNull(describeResponse, "Describe response should not be null"); + assertEquals(tableName, describeResponse.getTable(), "Table name should match in describe"); + assertNotNull(describeResponse.getSchema(), "Schema should not be null"); + assertNotNull(describeResponse.getStats(), "Stats should not be null"); + + // Verify schema + JsonSchema responseSchema = describeResponse.getSchema(); + assertNotNull(responseSchema, "Schema object should not be null"); + assertNotNull(responseSchema.getFields(), "Schema fields should not be null"); + assertEquals(3, responseSchema.getFields().size(), "Schema should have 3 fields"); + + List fieldNames = + responseSchema.getFields().stream().map(JsonField::getName).collect(Collectors.toList()); + assertTrue(fieldNames.contains("id"), "Schema should contain 'id' field"); + assertTrue(fieldNames.contains("name"), "Schema should contain 'name' field"); + assertTrue(fieldNames.contains("embedding"), "Schema should contain 'embedding' field"); + System.out.println("✓ Table schema verified with fields: " + fieldNames); + + // Verify version and stats + assertNotNull(describeResponse.getVersion(), "Version should not be null"); + assertTrue(describeResponse.getVersion() >= 1, "Version should be at least 1 for new table"); + assertTrue( + describeResponse.getStats().getNumFragments() >= 0, + "Number of fragments should be non-negative"); + System.out.println("✓ Table version: " + describeResponse.getVersion()); + System.out.println("✓ Table fragments: " + describeResponse.getStats().getNumFragments()); + + // Test insert table + System.out.println("\n--- Testing insert table ---"); + byte[] insertData1 = + new ArrowTestUtils.TableDataBuilder(allocator) + .addRows(1000, 2) // Start IDs from 1000 to differentiate + .build(); + + InsertTableResponse insertResponse = namespace.insertTable(tableName, insertData1, "append"); + assertNotNull(insertResponse, "Insert response should not be null"); + assertNotNull(insertResponse.getVersion(), "Insert response version should not be null"); + System.out.println("✓ Inserted 2 rows, new version: " + insertResponse.getVersion()); + + // Verify row count after first insert + long count2 = TestUtils.countRows(namespace, tableName); + assertEquals(5, count2, "Row count should be 5 after first insert"); + System.out.println("✓ Verified row count after first insert: " + count2); + + // Second insert + System.out.println("\n--- Testing second insert ---"); + byte[] insertData2 = + new ArrowTestUtils.TableDataBuilder(allocator) + .addRows(2000, 3) // Start IDs from 2000 + .build(); + + InsertTableResponse secondInsertResponse = + namespace.insertTable(tableName, insertData2, "append"); + assertNotNull(secondInsertResponse, "Second insert response should not be null"); + System.out.println( + "✓ Inserted 3 more rows, new version: " + secondInsertResponse.getVersion()); + + // Verify final row count + long finalCount = TestUtils.countRows(namespace, tableName); + assertEquals(8, finalCount, "Row count should be 8 after second insert"); + System.out.println("✓ Verified final row count: " + finalCount); + + System.out.println("\n✓ Table lifecycle test passed!"); + + } finally { + // Clean up + TestUtils.dropTable(namespace, tableName); + + // Verify table was dropped + System.out.println("\n--- Verifying table was dropped ---"); + try { + DescribeTableRequest verifyDropRequest = new DescribeTableRequest(); + verifyDropRequest.setName(tableName); + namespace.describeTable(verifyDropRequest); + fail("Expected exception when describing dropped table"); + } catch (LanceNamespaceException e) { + assertEquals(404, e.getCode(), "Should get 404 error code for non-existent table"); + System.out.println("✓ Confirmed table no longer exists (404 error code)"); + } + } + } + + @Test + public void testDescribeTableWithVersion() throws IOException { + skipIfNotConfigured(); + + System.out.println("\n=== Test: Describe Table With Version ==="); + String tableName = TestUtils.generateTableName("test_describe_version"); + + try { + // Create table + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 5).build(); + CreateTableResponse createResponse = namespace.createTable(tableName, tableData); + assertNotNull(createResponse, "Create response should not be null"); + + // Get initial version + DescribeTableRequest describeV1 = new DescribeTableRequest(); + describeV1.setName(tableName); + DescribeTableResponse v1Response = namespace.describeTable(describeV1); + Long version1 = v1Response.getVersion(); + System.out.println("Initial version: " + version1); + + // Insert more data to create new version + byte[] insertData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(100, 5).build(); + namespace.insertTable(tableName, insertData, "append"); + + // Describe current version + DescribeTableRequest describeCurrent = new DescribeTableRequest(); + describeCurrent.setName(tableName); + DescribeTableResponse currentResponse = namespace.describeTable(describeCurrent); + Long currentVersion = currentResponse.getVersion(); + System.out.println("Current version after insert: " + currentVersion); + assertTrue(currentVersion > version1, "Version should increase after insert"); + + // Describe specific older version + DescribeTableRequest describeOldVersion = new DescribeTableRequest(); + describeOldVersion.setName(tableName); + describeOldVersion.setVersion(version1); + DescribeTableResponse oldVersionResponse = namespace.describeTable(describeOldVersion); + + assertEquals(version1, oldVersionResponse.getVersion(), "Should return requested version"); + + // Verify nested structures in response + assertNotNull(oldVersionResponse.getSchema(), "Schema should not be null"); + assertNotNull(oldVersionResponse.getSchema().getFields(), "Schema fields should not be null"); + + // Check JsonField structure + for (JsonField field : oldVersionResponse.getSchema().getFields()) { + assertNotNull(field.getName(), "Field name should not be null"); + assertNotNull(field.getType(), "Field type should not be null"); + assertNotNull(field.getNullable(), "Field nullable should not be null"); + + // Check JsonDataType structure + JsonDataType dataType = field.getType(); + assertNotNull(dataType.getType(), "Data type name should not be null"); + + // For FixedSizeList (embedding field), check nested fields + if ("embedding".equals(field.getName())) { + assertNotNull(dataType.getFields(), "Embedding field should have nested fields"); + assertFalse(dataType.getFields().isEmpty(), "Embedding field should have item field"); + } + } + + // Verify TableBasicStats structure + TableBasicStats stats = oldVersionResponse.getStats(); + assertNotNull(stats, "Stats should not be null"); + assertNotNull(stats.getNumDeletedRows(), "Num deleted rows should not be null"); + assertNotNull(stats.getNumFragments(), "Num fragments should not be null"); + assertTrue(stats.getNumFragments() >= 0, "Num fragments should be non-negative"); + + System.out.println("✓ Describe table with version tested successfully"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableMergeInsertTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableMergeInsertTest.java new file mode 100644 index 000000000..e392ac6a6 --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableMergeInsertTest.java @@ -0,0 +1,199 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.table; + +import com.lancedb.lance.namespace.model.*; +import com.lancedb.lance.namespace.test.BaseNamespaceTest; +import com.lancedb.lance.namespace.test.utils.ArrowTestUtils; +import com.lancedb.lance.namespace.test.utils.TestUtils; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** Tests for merge insert (upsert) operations. */ +public class TableMergeInsertTest extends BaseNamespaceTest { + + @Test + public void testMergeInsert() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Merge Insert (Upsert) ==="); + String tableName = TestUtils.generateTableName("test_merge"); + + try { + // Step 1: Create table with 3 rows + System.out.println("\n--- Step 1: Creating table with 3 rows ---"); + byte[] tableData = + new ArrowTestUtils.TableDataBuilder(allocator) + .addRow(1, "Alice", generateVector(1)) + .addRow(2, "Bob", generateVector(2)) + .addRow(3, "Charlie", generateVector(3)) + .build(); + + CreateTableResponse createResponse = namespace.createTable(tableName, tableData); + assertNotNull(createResponse, "Create table response should not be null"); + + // Verify initial data + long initialCount = TestUtils.countRows(namespace, tableName); + assertEquals(3, initialCount, "Initial row count should be 3"); + + // Step 2: Merge insert with some matching and some new rows + System.out.println("\n--- Step 2: Merge insert with mixed rows ---"); + // Create batch with rows: id=2,3,4 (2,3 exist, 4 is new) + byte[] mergeData = + new ArrowTestUtils.TableDataBuilder(allocator) + .addRow(2, "Bob Updated", generateVector(20)) + .addRow(3, "Charlie Updated", generateVector(30)) + .addRow(4, "David", generateVector(4)) + .build(); + + // Perform merge insert + MergeInsertTableRequest mergeRequest = new MergeInsertTableRequest(); + mergeRequest.setName(tableName); + + MergeInsertTableResponse mergeResponse = + namespace.mergeInsertTable( + mergeRequest, + mergeData, + "id", // match on id column + true, // when_matched_update_all + true // when_not_matched_insert_all + ); + + assertNotNull(mergeResponse, "Merge response should not be null"); + assertEquals(2, mergeResponse.getNumUpdatedRows().longValue(), "Should have updated 2 rows"); + assertEquals(1, mergeResponse.getNumInsertedRows().longValue(), "Should have inserted 1 row"); + System.out.println( + "✓ Merge insert completed: " + + mergeResponse.getNumUpdatedRows() + + " updated, " + + mergeResponse.getNumInsertedRows() + + " inserted"); + + // Verify final count + long finalCount = TestUtils.countRows(namespace, tableName); + assertEquals(4, finalCount, "Final row count should be 4"); + + // Step 3: Verify the updates by querying the table + System.out.println("\n--- Step 3: Verifying merged data ---"); + QueryRequest queryRequest = TestUtils.createSimpleQuery(tableName, 4); + byte[] queryResult = namespace.queryTable(queryRequest); + assertNotNull(queryResult, "Query result should not be null"); + + // Extract and verify the data + List ids = ArrowTestUtils.extractColumn(queryResult, allocator, "id", Integer.class); + List names = + ArrowTestUtils.extractColumn(queryResult, allocator, "name", String.class); + + Map idToName = new HashMap<>(); + for (int i = 0; i < ids.size(); i++) { + idToName.put(ids.get(i), names.get(i)); + } + + // Verify the merged data + assertEquals(4, idToName.size(), "Should have 4 rows total"); + assertEquals("Alice", idToName.get(1), "ID 1 should remain unchanged"); + assertEquals("Bob Updated", idToName.get(2), "ID 2 should be updated"); + assertEquals("Charlie Updated", idToName.get(3), "ID 3 should be updated"); + assertEquals("David", idToName.get(4), "ID 4 should be new"); + System.out.println("✓ Merge insert data verified successfully"); + + // Test merge with only inserts + System.out.println("\n--- Step 4: Testing merge with only new rows ---"); + byte[] insertOnlyData = + new ArrowTestUtils.TableDataBuilder(allocator) + .addRow(5, "Eve", generateVector(5)) + .addRow(6, "Frank", generateVector(6)) + .build(); + + MergeInsertTableRequest insertOnlyRequest = new MergeInsertTableRequest(); + insertOnlyRequest.setName(tableName); + + MergeInsertTableResponse insertOnlyResponse = + namespace.mergeInsertTable(insertOnlyRequest, insertOnlyData, "id", true, true); + + assertEquals( + 0, insertOnlyResponse.getNumUpdatedRows().longValue(), "Should have updated 0 rows"); + assertEquals( + 2, insertOnlyResponse.getNumInsertedRows().longValue(), "Should have inserted 2 rows"); + System.out.println("✓ Insert-only merge completed successfully"); + + long finalCount2 = TestUtils.countRows(namespace, tableName); + assertEquals(6, finalCount2, "Final row count should be 6"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testMergeInsertWithUpdateOnly() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Merge Insert with Update Only ==="); + String tableName = TestUtils.generateTableName("test_merge_update_only"); + + try { + // Create initial table + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 5).build(); + namespace.createTable(tableName, tableData); + + // Try to merge with when_not_matched_insert_all = false + byte[] mergeData = + new ArrowTestUtils.TableDataBuilder(allocator) + .addRow(3, "Updated Three", generateVector(30)) + .addRow(6, "New Six", generateVector(6)) // This should NOT be inserted + .build(); + + MergeInsertTableRequest mergeRequest = new MergeInsertTableRequest(); + mergeRequest.setName(tableName); + + MergeInsertTableResponse mergeResponse = + namespace.mergeInsertTable( + mergeRequest, + mergeData, + "id", + true, // when_matched_update_all + false // when_not_matched_insert_all = false + ); + + assertEquals(1, mergeResponse.getNumUpdatedRows().longValue(), "Should have updated 1 row"); + assertEquals( + 0, mergeResponse.getNumInsertedRows().longValue(), "Should have inserted 0 rows"); + + // Verify count remains 5 + long finalCount = TestUtils.countRows(namespace, tableName); + assertEquals(5, finalCount, "Row count should remain 5"); + + System.out.println("✓ Update-only merge completed successfully"); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + private static float[] generateVector(int seed) { + float[] vector = new float[128]; + for (int i = 0; i < 128; i++) { + vector[i] = seed * 10.0f + i * 0.1f; + } + return vector; + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableUpdateDeleteTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableUpdateDeleteTest.java new file mode 100644 index 000000000..691967486 --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableUpdateDeleteTest.java @@ -0,0 +1,151 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.table; + +import com.lancedb.lance.namespace.model.*; +import com.lancedb.lance.namespace.test.BaseNamespaceTest; +import com.lancedb.lance.namespace.test.utils.ArrowTestUtils; +import com.lancedb.lance.namespace.test.utils.TestUtils; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** Tests for table update and delete operations. */ +public class TableUpdateDeleteTest extends BaseNamespaceTest { + + @Test + public void testUpdateTable() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Update Table ==="); + String tableName = TestUtils.generateTableName("test_update"); + + try { + // Create table with 3 rows (IDs: 1, 2, 3) + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 3).build(); + CreateTableResponse createResponse = namespace.createTable(tableName, tableData); + assertNotNull(createResponse, "Create table response should not be null"); + + // Update all rows (id = id + 1) + System.out.println("\n--- Step 2: Updating all rows (id = id + 1) ---"); + UpdateTableRequest updateRequest = new UpdateTableRequest(); + updateRequest.setName(tableName); + updateRequest.setNamespace(new ArrayList<>()); + List> updates = new ArrayList<>(); + updates.add(Arrays.asList("id", "id + 1")); + updateRequest.setUpdates(updates); + + UpdateTableResponse updateResponse = namespace.updateTable(updateRequest); + assertNotNull(updateResponse, "Update response should not be null"); + assertEquals(3, updateResponse.getUpdatedRows().longValue(), "Should have updated 3 rows"); + System.out.println("✓ Updated 3 rows, IDs should now be: 2, 3, 4"); + + // Update rows with predicate (id > 2) + System.out.println("\n--- Step 3: Updating rows with predicate (id > 2) ---"); + UpdateTableRequest predicateUpdateRequest = new UpdateTableRequest(); + predicateUpdateRequest.setName(tableName); + predicateUpdateRequest.setNamespace(new ArrayList<>()); + predicateUpdateRequest.setPredicate("id > 2"); + List> predicateUpdates = new ArrayList<>(); + predicateUpdates.add(Arrays.asList("id", "id + 10")); + predicateUpdateRequest.setUpdates(predicateUpdates); + + UpdateTableResponse predicateUpdateResponse = namespace.updateTable(predicateUpdateRequest); + assertNotNull(predicateUpdateResponse, "Predicate update response should not be null"); + assertEquals( + 2, predicateUpdateResponse.getUpdatedRows().longValue(), "Should have updated 2 rows"); + System.out.println("✓ Updated 2 rows, IDs should now be: 2, 13, 14"); + + // Verify final state + QueryRequest queryRequest = TestUtils.createSimpleQuery(tableName, 3); + byte[] queryResult = namespace.queryTable(queryRequest); + assertNotNull(queryResult, "Query result should not be null"); + + List idValues = + ArrowTestUtils.extractColumn(queryResult, allocator, "id", Integer.class); + Collections.sort(idValues); + assertEquals(3, idValues.size(), "Should have exactly 3 rows"); + assertEquals( + Arrays.asList(2, 13, 14), idValues, "ID values should be [2, 13, 14] after updates"); + System.out.println("✓ Verified final ID values: " + idValues); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } + + @Test + public void testDeleteFromTable() throws IOException { + skipIfNotConfigured(); + + System.out.println("=== Test: Delete From Table ==="); + String tableName = TestUtils.generateTableName("test_delete"); + + try { + // Create table with 3 rows + System.out.println("\n--- Step 1: Creating table with 3 rows ---"); + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 3).build(); + CreateTableResponse createResponse = namespace.createTable(tableName, tableData); + assertNotNull(createResponse, "Create table response should not be null"); + + long initialCount = TestUtils.countRows(namespace, tableName); + assertEquals(3, initialCount, "Initial row count should be 3"); + + // Delete row with id=1 + System.out.println("\n--- Step 2: Deleting row with id=1 ---"); + DeleteFromTableRequest deleteRequest = new DeleteFromTableRequest(); + deleteRequest.setName(tableName); + deleteRequest.setPredicate("id = 1"); + + DeleteFromTableResponse deleteResponse = namespace.deleteFromTable(deleteRequest); + assertEquals(2, deleteResponse.getVersion(), "Version should increment"); + + long countAfterFirst = TestUtils.countRows(namespace, tableName); + assertEquals(2, countAfterFirst, "Should have 2 rows after deleting id=1"); + System.out.println("✓ Deleted 1 row, remaining count: " + countAfterFirst); + + // Delete with complex predicate (id > 2) + System.out.println("\n--- Step 3: Deleting with complex predicate (id > 2) ---"); + DeleteFromTableRequest complexDeleteRequest = new DeleteFromTableRequest(); + complexDeleteRequest.setName(tableName); + complexDeleteRequest.setPredicate("id > 2"); + + DeleteFromTableResponse complexDeleteResponse = + namespace.deleteFromTable(complexDeleteRequest); + assertEquals(3, complexDeleteResponse.getVersion(), "Version should increment again"); + + long finalCount = TestUtils.countRows(namespace, tableName); + assertEquals(1, finalCount, "Should have 1 row after deleting id > 2"); + System.out.println("✓ Deleted rows where id > 2, remaining count: " + finalCount); + + // Verify remaining row + QueryRequest queryRequest = TestUtils.createSimpleQuery(tableName, 1); + byte[] queryResult = namespace.queryTable(queryRequest); + List remainingIds = + ArrowTestUtils.extractColumn(queryResult, allocator, "id", Integer.class); + assertEquals(Arrays.asList(2), remainingIds, "Only ID 2 should remain"); + System.out.println("✓ Verified remaining row has ID: " + remainingIds.get(0)); + + } finally { + TestUtils.dropTable(namespace, tableName); + } + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/ArrowTestUtils.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/ArrowTestUtils.java new file mode 100644 index 000000000..e54389802 --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/ArrowTestUtils.java @@ -0,0 +1,366 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.utils; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.function.Consumer; + +/** + * Utility class for creating and manipulating Arrow data in tests. Provides builder-style methods + * for easy test data creation. + */ +public class ArrowTestUtils { + + /** Builder for creating Arrow data with common table schema. */ + public static class TableDataBuilder { + private final BufferAllocator allocator; + private final List rows = new ArrayList<>(); + private Schema customSchema; + private Map customTexts = new HashMap<>(); + + // Default sample names for test data + private static final String[] DEFAULT_NAMES = { + "Alice", "Bob", "Charlie", "David", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack", + "Kate", "Liam", "Maya", "Noah", "Olivia", "Peter", "Quinn", "Rose", "Sam", "Tara" + }; + + public TableDataBuilder(BufferAllocator allocator) { + this.allocator = allocator; + } + + /** Add a row with default values. */ + public TableDataBuilder addRow(int id) { + return addRow(id, DEFAULT_NAMES[id % DEFAULT_NAMES.length], generateVector(id, 128)); + } + + /** Add a row with custom values. */ + public TableDataBuilder addRow(int id, String name, float[] embedding) { + rows.add(new TableRow(id, name, embedding)); + return this; + } + + /** Add multiple rows with default values. */ + public TableDataBuilder addRows(int startId, int count) { + for (int i = 0; i < count; i++) { + addRow(startId + i); + } + return this; + } + + /** Set a custom schema instead of the default one. */ + public TableDataBuilder withSchema(Schema schema) { + this.customSchema = schema; + return this; + } + + /** Set custom text for a specific row ID. */ + public TableDataBuilder withText(int id, String text) { + this.customTexts.put(id, text); + return this; + } + + /** Build the Arrow IPC data. */ + public byte[] build() throws IOException { + Schema schema = customSchema != null ? customSchema : createDefaultSchema(); + + try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + root.setRowCount(rows.size()); + + // Populate vectors based on schema + for (Field field : schema.getFields()) { + populateVector(root, field, rows); + } + + // Serialize to Arrow IPC format + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ArrowStreamWriter writer = + new ArrowStreamWriter(root, null, Channels.newChannel(out))) { + writer.start(); + writer.writeBatch(); + writer.end(); + } + + return out.toByteArray(); + } + } + + private void populateVector(VectorSchemaRoot root, Field field, List rows) { + String fieldName = field.getName(); + + switch (fieldName) { + case "id": + IntVector idVector = (IntVector) root.getVector("id"); + for (int i = 0; i < rows.size(); i++) { + idVector.setSafe(i, rows.get(i).id); + } + idVector.setValueCount(rows.size()); + break; + + case "name": + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + for (int i = 0; i < rows.size(); i++) { + nameVector.setSafe(i, rows.get(i).name.getBytes(StandardCharsets.UTF_8)); + } + nameVector.setValueCount(rows.size()); + break; + + case "text": + VarCharVector textVector = (VarCharVector) root.getVector("text"); + for (int i = 0; i < rows.size(); i++) { + int rowId = rows.get(i).id; + String text; + if (customTexts.containsKey(rowId)) { + text = customTexts.get(rowId); + } else { + // Default text if not specified + text = "Default text for row " + rowId; + } + textVector.setSafe(i, text.getBytes(StandardCharsets.UTF_8)); + } + textVector.setValueCount(rows.size()); + break; + + case "category": + VarCharVector categoryVector = (VarCharVector) root.getVector("category"); + String[] categories = {"category1", "category2", "category3"}; + for (int i = 0; i < rows.size(); i++) { + // Use modulo to evenly distribute categories + String category = categories[rows.get(i).id % 3]; + categoryVector.setSafe(i, category.getBytes(StandardCharsets.UTF_8)); + } + categoryVector.setValueCount(rows.size()); + break; + + case "embedding": + FixedSizeListVector vectorVector = (FixedSizeListVector) root.getVector("embedding"); + Float4Vector dataVector = (Float4Vector) vectorVector.getDataVector(); + vectorVector.allocateNew(); + + for (int row = 0; row < rows.size(); row++) { + vectorVector.setNotNull(row); + float[] embedding = rows.get(row).embedding; + for (int dim = 0; dim < embedding.length; dim++) { + int index = row * embedding.length + dim; + dataVector.setSafe(index, embedding[dim]); + } + } + + dataVector.setValueCount(rows.size() * rows.get(0).embedding.length); + vectorVector.setValueCount(rows.size()); + break; + } + } + + private static float[] generateVector(int seed, int dimensions) { + float[] vector = new float[dimensions]; + // Create deterministic vectors: each vector has all elements set to the row id value + // This makes search results predictable: searching for vector of all 10s will find + // row 10 as closest, then 11, then 9, etc. + for (int i = 0; i < dimensions; i++) { + vector[i] = (float) seed; + } + return vector; + } + } + + /** Create a default schema with id, name, text, and embedding fields. */ + public static Schema createDefaultSchema() { + return createDefaultSchema(128); + } + + /** + * Create a default schema with id, name, category, and embedding fields for general test cases. + */ + public static Schema createDefaultSchema(int embeddingDimension) { + Field idField = new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field nameField = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + Field categoryField = new Field("category", FieldType.nullable(new ArrowType.Utf8()), null); + Field embeddingField = + new Field( + "embedding", + FieldType.nullable(new ArrowType.FixedSizeList(embeddingDimension)), + Arrays.asList( + new Field( + "item", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null))); + + return new Schema(Arrays.asList(idField, nameField, categoryField, embeddingField)); + } + + /** Create a schema with text field for FTS tests. */ + public static Schema createSchemaWithText(int embeddingDimension) { + Field idField = new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field nameField = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + Field textField = new Field("text", FieldType.nullable(new ArrowType.Utf8()), null); + Field categoryField = new Field("category", FieldType.nullable(new ArrowType.Utf8()), null); + Field embeddingField = + new Field( + "embedding", + FieldType.nullable(new ArrowType.FixedSizeList(embeddingDimension)), + Arrays.asList( + new Field( + "item", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null))); + + return new Schema(Arrays.asList(idField, nameField, textField, categoryField, embeddingField)); + } + + /** Read Arrow file format data and process it. */ + public static void readArrowFile( + byte[] data, BufferAllocator allocator, Consumer processor) + throws IOException { + ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(data); + try (ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { + for (int i = 0; i < reader.getRecordBlocks().size(); i++) { + reader.loadRecordBatch(reader.getRecordBlocks().get(i)); + processor.accept(reader.getVectorSchemaRoot()); + } + } + } + + /** Count total rows in Arrow file format data. */ + public static int countRows(byte[] data, BufferAllocator allocator) throws IOException { + int[] totalRows = {0}; + readArrowFile(data, allocator, root -> totalRows[0] += root.getRowCount()); + return totalRows[0]; + } + + /** Extract values from a specific column. */ + public static List extractColumn( + byte[] data, BufferAllocator allocator, String columnName, Class type) throws IOException { + List values = new ArrayList<>(); + + readArrowFile( + data, + allocator, + root -> { + if (type == Integer.class) { + IntVector vector = (IntVector) root.getVector(columnName); + for (int i = 0; i < root.getRowCount(); i++) { + if (!vector.isNull(i)) { + values.add(type.cast(vector.get(i))); + } + } + } else if (type == String.class) { + VarCharVector vector = (VarCharVector) root.getVector(columnName); + for (int i = 0; i < root.getRowCount(); i++) { + if (!vector.isNull(i)) { + values.add(type.cast(new String(vector.get(i), StandardCharsets.UTF_8))); + } + } + } + // Add more type handlers as needed + }); + + return values; + } + + /** Simple row representation for building test data. */ + private static class TableRow { + final int id; + final String name; + final float[] embedding; + + TableRow(int id, String name, float[] embedding) { + this.id = id; + this.name = name; + this.embedding = embedding; + } + } + + /** SeekableByteChannel implementation for reading Arrow file format from byte array. */ + public static class ByteArraySeekableByteChannel implements SeekableByteChannel { + private final byte[] data; + private long position = 0; + private boolean isOpen = true; + + public ByteArraySeekableByteChannel(byte[] data) { + this.data = data; + } + + @Override + public long position() throws IOException { + return position; + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + if (newPosition < 0 || newPosition > data.length) { + throw new IOException("Invalid position: " + newPosition); + } + position = newPosition; + return this; + } + + @Override + public long size() throws IOException { + return data.length; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + if (!isOpen) { + throw new IOException("Channel is closed"); + } + int remaining = dst.remaining(); + int available = (int) (data.length - position); + if (available <= 0) { + return -1; + } + int toRead = Math.min(remaining, available); + dst.put(data, (int) position, toRead); + position += toRead; + return toRead; + } + + @Override + public int write(ByteBuffer src) throws IOException { + throw new UnsupportedOperationException("Read-only channel"); + } + + @Override + public SeekableByteChannel truncate(long size) throws IOException { + throw new UnsupportedOperationException("Read-only channel"); + } + + @Override + public boolean isOpen() { + return isOpen; + } + + @Override + public void close() throws IOException { + isOpen = false; + } + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java new file mode 100644 index 000000000..f7cdba44f --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java @@ -0,0 +1,144 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.utils; + +import com.lancedb.lance.namespace.LanceRestNamespace; +import com.lancedb.lance.namespace.model.*; + +import java.util.Optional; +import java.util.UUID; + +/** Common test utilities for Lance Namespace tests. */ +public class TestUtils { + + /** Generate a unique table name for testing. */ + public static String generateTableName(String prefix) { + return prefix + "_" + UUID.randomUUID().toString().replace("-", "_").substring(0, 8); + } + + /** Generate a unique table name with default prefix. */ + public static String generateTableName() { + return generateTableName("test_table"); + } + + /** Wait for an index to be created. */ + public static boolean waitForIndex( + LanceRestNamespace namespace, String tableName, String indexName, int maxSeconds) + throws InterruptedException { + IndexListRequest listRequest = new IndexListRequest(); + listRequest.setName(tableName); + + for (int i = 0; i < maxSeconds; i++) { + IndexListResponse listResponse = namespace.listIndices(listRequest); + if (listResponse.getIndexes() != null + && listResponse.getIndexes().stream() + .anyMatch(idx -> idx.getIndexName().equals(indexName))) { + return true; + } + Thread.sleep(1000); + } + return false; + } + + /** Wait for an index to be fully built with no unindexed rows. */ + public static boolean waitForIndexComplete( + LanceRestNamespace namespace, String tableName, String indexName, int maxSeconds) + throws InterruptedException { + IndexListRequest listRequest = new IndexListRequest(); + listRequest.setName(tableName); + + for (int i = 0; i < maxSeconds; i++) { + IndexListResponse listResponse = namespace.listIndices(listRequest); + if (listResponse.getIndexes() != null) { + Optional indexOpt = + listResponse.getIndexes().stream() + .filter(idx -> idx.getIndexName().equals(indexName)) + .findFirst(); + + if (indexOpt.isPresent()) { + // Index exists, now check if it's fully built + IndexStatsRequest statsRequest = new IndexStatsRequest(); + statsRequest.setName(tableName); + + IndexStatsResponse stats = namespace.getIndexStats(statsRequest, indexName); + if (stats != null + && stats.getNumUnindexedRows() != null + && stats.getNumUnindexedRows() == 0) { + System.out.println("✓ Index " + indexName + " is fully built with 0 unindexed rows"); + return true; + } else if (stats != null && stats.getNumUnindexedRows() != null) { + System.out.println( + " Waiting for index... " + stats.getNumUnindexedRows() + " rows remaining"); + } + } + } + Thread.sleep(1000); + } + return false; + } + + /** Drop a table and print confirmation. */ + public static DropTableResponse dropTable(LanceRestNamespace namespace, String tableName) { + DropTableRequest dropRequest = new DropTableRequest(); + dropRequest.setName(tableName); + + DropTableResponse response = namespace.dropTable(dropRequest); + System.out.println("✓ Table dropped successfully: " + tableName); + + return response; + } + + /** Count rows in a table. */ + public static long countRows(LanceRestNamespace namespace, String tableName) { + CountRowsRequest countRequest = new CountRowsRequest(); + countRequest.setName(tableName); + return namespace.countRows(countRequest); + } + + /** Create a simple query request for testing. */ + public static QueryRequest createSimpleQuery(String tableName, int k) { + QueryRequest query = new QueryRequest(); + query.setName(tableName); + query.setK(k); + // Add default columns to avoid "no columns selected" error + query.setColumns(java.util.Arrays.asList("id", "name", "category", "embedding")); + return query; + } + + /** Create a vector query request with a specific target value. */ + public static QueryRequest createVectorQuery(String tableName, int k, int dimensions) { + return createVectorQuery(tableName, k, dimensions, 10.0f); + } + + /** Create a vector query request with a specific target value for all dimensions. */ + public static QueryRequest createVectorQuery( + String tableName, int k, int dimensions, float targetValue) { + QueryRequest query = new QueryRequest(); + query.setName(tableName); + query.setK(k); + + // Generate a vector with all elements set to targetValue + // This will find rows where the embedding values are closest to targetValue + java.util.List vector = new java.util.ArrayList<>(); + for (int i = 0; i < dimensions; i++) { + vector.add(targetValue); + } + query.setVector(vector); + + // Add default columns to avoid "no columns selected" error + query.setColumns(java.util.Arrays.asList("id", "name", "category", "embedding")); + + return query; + } +} From a83995d230a3a980b0b3084822e43ed9b4065f1b Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sun, 20 Jul 2025 11:36:36 -0700 Subject: [PATCH 05/10] Update java sdk doc --- docs/src/user-guide/java-sdk.md | 397 ++++++++++++++++++++++---------- 1 file changed, 275 insertions(+), 122 deletions(-) diff --git a/docs/src/user-guide/java-sdk.md b/docs/src/user-guide/java-sdk.md index 6322662f0..53f424ff9 100644 --- a/docs/src/user-guide/java-sdk.md +++ b/docs/src/user-guide/java-sdk.md @@ -97,70 +97,66 @@ For detailed request/response structures, refer to the [Apache Client documentat ### Creating a Table -LanceDB uses Apache Arrow format for data exchange. Arrow provides: -- Cross-language compatibility -- Rich data type support including nested types and tensors +LanceDB uses Apache Arrow format for data exchange. Here's a simple example creating a table with ID and embedding columns: ```java import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.types.pojo.*; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.ipc.ArrowStreamWriter; import java.io.ByteArrayOutputStream; import java.nio.channels.Channels; -import java.nio.charset.StandardCharsets; import java.util.Arrays; -// Define Arrow schema +// Define schema: id and embedding columns Field idField = new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null); -Field nameField = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); Field embeddingField = new Field( "embedding", - FieldType.nullable(new ArrowType.FixedSizeList(128)), + FieldType.nullable(new ArrowType.FixedSizeList(128)), // 128-dimensional vectors Arrays.asList( new Field("item", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), null) ) ); -Schema schema = new Schema(Arrays.asList(idField, nameField, embeddingField)); +Schema schema = new Schema(Arrays.asList(idField, embeddingField)); -// Create data +// Create table with 1000 rows try (BufferAllocator allocator = new RootAllocator(); VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + int numRows = 1000; + root.setRowCount(numRows); + + // Get vectors IntVector idVector = (IntVector) root.getVector("id"); - VarCharVector nameVector = (VarCharVector) root.getVector("name"); - FixedSizeListVector vectorVector = (FixedSizeListVector) root.getVector("embedding"); + FixedSizeListVector embeddingVector = (FixedSizeListVector) root.getVector("embedding"); + Float4Vector dataVector = (Float4Vector) embeddingVector.getDataVector(); - // Set row count - root.setRowCount(3); + // Allocate memory + embeddingVector.allocateNew(); // Populate data - for (int i = 0; i < 3; i++) { + for (int i = 0; i < numRows; i++) { + // Set ID idVector.setSafe(i, i + 1); - nameVector.setSafe(i, ("User" + i).getBytes(StandardCharsets.UTF_8)); - } - - // Populate embeddings - Float4Vector dataVector = (Float4Vector) vectorVector.getDataVector(); - vectorVector.allocateNew(); - - for (int row = 0; row < 3; row++) { - vectorVector.setNotNull(row); + + // Set embedding vector + embeddingVector.setNotNull(i); for (int dim = 0; dim < 128; dim++) { - int index = row * 128 + dim; + int index = i * 128 + dim; + // Example: random values (in practice, use your actual embeddings) dataVector.setSafe(index, (float) Math.random()); } } // Set value counts - idVector.setValueCount(3); - nameVector.setValueCount(3); - dataVector.setValueCount(3 * 128); - vectorVector.setValueCount(3); + idVector.setValueCount(numRows); + dataVector.setValueCount(numRows * 128); + embeddingVector.setValueCount(numRows); // Serialize to Arrow IPC format ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -170,12 +166,15 @@ try (BufferAllocator allocator = new RootAllocator(); writer.end(); } - // Create table - byte[] arrowIpcData = out.toByteArray(); - CreateTableResponse response = namespace.createTable("my_table", arrowIpcData); + // Create table in LanceDB + byte[] arrowData = out.toByteArray(); + CreateTableResponse response = namespace.createTable("my_vectors", arrowData); + System.out.println("Created table with " + numRows + " rows"); } ``` +For more complex schemas (e.g., with text fields for full-text search, categorical fields for filtering), you can add additional fields to the schema as needed. + ### Querying a Table Query results are returned in Arrow File format. Use `ArrowFileReader` to read the results. @@ -187,55 +186,49 @@ Query results are returned in Arrow File format. Use `ArrowFileReader` to read t ```java import com.lancedb.lance.namespace.model.QueryRequest; -import com.lancedb.lance.namespace.model.QueryRequestVector; import org.apache.arrow.vector.ipc.ArrowFileReader; -import org.apache.arrow.vector.ipc.ArrowBlock; -import org.apache.arrow.vector.ipc.SeekableReadChannel; -import java.io.ByteArrayInputStream; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import java.nio.channels.SeekableByteChannel; import java.util.List; import java.util.ArrayList; import java.util.Arrays; -// Create query request +// Find similar items by vector QueryRequest queryRequest = new QueryRequest(); -queryRequest.setName("my_table"); +queryRequest.setName("my_vectors"); +queryRequest.setK(10); // Get top 10 results -// Single vector search (most common use case) +// Create query vector (in practice, this would be your actual query embedding) List queryVector = new ArrayList<>(); for (int i = 0; i < 128; i++) { queryVector.add((float) Math.random()); } - -// For single vector, you can set it directly queryRequest.setVector(queryVector); -queryRequest.setK(5); // Get top 5 results // REQUIRED: Specify columns to return -queryRequest.setColumns(Arrays.asList("id", "name", "embedding")); +queryRequest.setColumns(Arrays.asList("id", "embedding")); + +// Optional: Set fast_search for better performance (only searches indexed data) +queryRequest.setFastSearch(true); // Execute query byte[] queryResult = namespace.queryTable(queryRequest); -// Parse Arrow result +// Parse results try (BufferAllocator allocator = new RootAllocator(); ArrowFileReader reader = new ArrowFileReader( - new SeekableReadChannel(Channels.newChannel(new ByteArrayInputStream(queryResult))), + new SeekableByteChannel() { /* See ArrowTestUtils for full implementation */ }, allocator)) { - // Process results - for (ArrowBlock block : reader.getRecordBlocks()) { - reader.loadRecordBatch(block); - VectorSchemaRoot root = reader.getVectorSchemaRoot(); - - // Access data from vectors - IntVector idVector = (IntVector) root.getVector("id"); - VarCharVector nameVector = (VarCharVector) root.getVector("name"); - - for (int i = 0; i < root.getRowCount(); i++) { - int id = idVector.get(i); - String name = new String(nameVector.get(i), StandardCharsets.UTF_8); - System.out.println("ID: " + id + ", Name: " + name); - } + reader.loadRecordBatch(reader.getRecordBlocks().get(0)); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + + IntVector idVector = (IntVector) root.getVector("id"); + + System.out.println("Found " + root.getRowCount() + " similar vectors:"); + for (int i = 0; i < Math.min(5, root.getRowCount()); i++) { + System.out.println(" ID: " + idVector.get(i)); } } ``` @@ -258,22 +251,45 @@ queryRequest.setFastSearch(true); // Recommended for better performance You can use SQL filters with or without vector search: ```java -// SQL filter only (no vector search) -QueryRequest filterOnlyQuery = new QueryRequest(); -filterOnlyQuery.setName("my_table"); -filterOnlyQuery.setK(10); -filterOnlyQuery.setFilter("category = 'electronics' AND price < 500"); -filterOnlyQuery.setFastSearch(true); -filterOnlyQuery.setColumns(Arrays.asList("id", "name", "category", "price")); // Required! - -// Vector search with SQL filter +// Example 1: Filter-only query (no vector search) +QueryRequest filterQuery = new QueryRequest(); +filterQuery.setName("my_table"); +filterQuery.setK(20); // Maximum results to return +filterQuery.setFilter("id >= 100 AND id < 200"); +filterQuery.setColumns(Arrays.asList("id")); // Required! + +byte[] filterResult = namespace.queryTable(filterQuery); + +// Example 2: Vector search with filter QueryRequest vectorWithFilter = new QueryRequest(); -vectorWithFilter.setName("my_table"); +vectorWithFilter.setName("my_vectors"); +vectorWithFilter.setK(5); + +// Create query vector +List queryVector = new ArrayList<>(); +for (int i = 0; i < 128; i++) { + queryVector.add((float) Math.random()); +} vectorWithFilter.setVector(queryVector); -vectorWithFilter.setK(10); -vectorWithFilter.setFilter("status = 'active' AND price < 1000"); -vectorWithFilter.setFastSearch(true); -vectorWithFilter.setColumns(Arrays.asList("id", "name", "status", "price")); // Required! + +// Only search within specific ID range +vectorWithFilter.setFilter("id >= 500 AND id < 600"); +vectorWithFilter.setColumns(Arrays.asList("id")); + +byte[] filteredVectorResult = namespace.queryTable(vectorWithFilter); + +// Example 3: Complex filter expressions +QueryRequest complexFilter = new QueryRequest(); +complexFilter.setName("my_table"); +complexFilter.setK(100); +complexFilter.setFilter("id >= 10 AND id <= 90"); +complexFilter.setColumns(Arrays.asList("id")); + +// Supported SQL operators: +// - Comparison: =, !=, <, >, <=, >= +// - Logical: AND, OR, NOT +// - IN: category IN ('category1', 'category2') +// - String: LIKE for pattern matching ``` ##### Prefilter vs Postfilter @@ -307,121 +323,211 @@ postfilterQuery.setFastSearch(true); LanceDB supports full-text search on string columns. First create an FTS index, then use text queries: ```java -// Create FTS index +// Step 1: Create table with text content (add text columns to your schema) + +// Step 2: Create FTS index CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); -ftsIndexRequest.setName("my_table"); +ftsIndexRequest.setName("documents"); ftsIndexRequest.setColumn("content"); ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); CreateIndexResponse ftsResponse = namespace.createIndex(ftsIndexRequest); -waitForIndexComplete("my_table", "content_idx", 30); +// Wait for index to be built +boolean indexReady = waitForIndexComplete("documents", "content_idx", 30); -// Perform full-text search +// Step 3: Perform full-text search import com.lancedb.lance.namespace.model.StringFtsQuery; import com.lancedb.lance.namespace.model.QueryRequestFullTextQuery; +// Example 1: Simple keyword search QueryRequest textQuery = new QueryRequest(); -textQuery.setName("my_table"); -textQuery.setK(10); +textQuery.setName("documents"); +textQuery.setK(5); +textQuery.setColumns(Arrays.asList("id", "title", "content")); // Required! -// Simple string query QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); StringFtsQuery fts = new StringFtsQuery(); -fts.setQuery("search terms"); // The search query -fts.setColumns(Arrays.asList("content", "title")); // Optional: columns to search +fts.setQuery("machine learning"); // Search for documents about machine learning fullTextQuery.setStringQuery(fts); textQuery.setFullTextQuery(fullTextQuery); byte[] results = namespace.queryTable(textQuery); +// Expected: Documents containing "machine" and/or "learning" + +// Example 2: Search specific columns +StringFtsQuery columnSearch = new StringFtsQuery(); +columnSearch.setQuery("neural networks"); +columnSearch.setColumns(Arrays.asList("content")); // Only search in content column +fullTextQuery.setStringQuery(columnSearch); + +// Example 3: Full-text search with filter +QueryRequest ftsWithFilter = new QueryRequest(); +ftsWithFilter.setName("documents"); +ftsWithFilter.setK(10); +ftsWithFilter.setFilter("id <= 3"); // Only search in first 3 documents +ftsWithFilter.setColumns(Arrays.asList("id", "title", "content")); + +StringFtsQuery filteredFts = new StringFtsQuery(); +filteredFts.setQuery("learning"); +QueryRequestFullTextQuery filteredFullText = new QueryRequestFullTextQuery(); +filteredFullText.setStringQuery(filteredFts); +ftsWithFilter.setFullTextQuery(filteredFullText); + +byte[] filteredResults = namespace.queryTable(ftsWithFilter); +// Expected: Documents 1-3 that contain "learning" ``` ##### Advanced: Structured Full-Text Search -The Java SDK now supports complex structured full-text queries including boolean queries, phrase queries, and boosted queries: +The Java SDK supports complex structured full-text queries including boolean queries and phrase queries: ```java import com.lancedb.lance.namespace.model.*; -// Boolean query example: (must contain "important" AND should contain "feature" OR "update") -QueryRequest structuredQuery = new QueryRequest(); -structuredQuery.setName("my_table"); -structuredQuery.setK(10); +// Example 1: Boolean Query - Complex search logic +QueryRequest booleanSearchQuery = new QueryRequest(); +booleanSearchQuery.setName("documents"); +booleanSearchQuery.setK(10); +booleanSearchQuery.setColumns(Arrays.asList("id", "title", "content")); +// Create structured query wrapper QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); StructuredFtsQuery structured = new StructuredFtsQuery(); FtsQuery ftsQuery = new FtsQuery(); -// Create a boolean query +// Boolean query: MUST contain "learning" AND (SHOULD contain "machine" OR "deep") BooleanQuery boolQuery = new BooleanQuery(); -// Must clause: documents must contain "important" +// MUST clause: documents must contain "learning" FtsQuery mustQuery = new FtsQuery(); MatchQuery mustMatch = new MatchQuery(); -mustMatch.setTerms("important"); +mustMatch.setTerms("learning"); mustMatch.setColumn("content"); mustQuery.setMatch(mustMatch); boolQuery.setMust(Arrays.asList(mustQuery)); -// Should clauses: documents should contain "feature" OR "update" +// SHOULD clauses: prefer documents with "machine" or "deep" FtsQuery shouldQuery1 = new FtsQuery(); MatchQuery shouldMatch1 = new MatchQuery(); -shouldMatch1.setTerms("feature"); +shouldMatch1.setTerms("machine"); +shouldMatch1.setColumn("content"); shouldQuery1.setMatch(shouldMatch1); FtsQuery shouldQuery2 = new FtsQuery(); MatchQuery shouldMatch2 = new MatchQuery(); -shouldMatch2.setTerms("update"); +shouldMatch2.setTerms("deep"); +shouldMatch2.setColumn("content"); shouldQuery2.setMatch(shouldMatch2); boolQuery.setShould(Arrays.asList(shouldQuery1, shouldQuery2)); -// Must NOT clause: exclude documents with "deprecated" +// Optional: MUST NOT clause FtsQuery mustNotQuery = new FtsQuery(); MatchQuery mustNotMatch = new MatchQuery(); -mustNotMatch.setTerms("deprecated"); +mustNotMatch.setTerms("beginner"); // Exclude beginner content mustNotQuery.setMatch(mustNotMatch); boolQuery.setMustNot(Arrays.asList(mustNotQuery)); +// Set the boolean query ftsQuery.setBoolean(boolQuery); structured.setQuery(ftsQuery); fullTextQuery.setStructuredQuery(structured); -structuredQuery.setFullTextQuery(fullTextQuery); - -byte[] boolResults = namespace.queryTable(structuredQuery); +booleanSearchQuery.setFullTextQuery(fullTextQuery); + +byte[] boolResults = namespace.queryTable(booleanSearchQuery); +// Expected: Documents containing "learning" (required) and preferably "machine" or "deep" + +// Example 2: Phrase Query - Find exact phrases +QueryRequest phraseSearchQuery = new QueryRequest(); +phraseSearchQuery.setName("documents"); +phraseSearchQuery.setK(5); +phraseSearchQuery.setColumns(Arrays.asList("id", "title", "content")); + +// Create phrase query +QueryRequestFullTextQuery phraseFullText = new QueryRequestFullTextQuery(); +StructuredFtsQuery phraseStructured = new StructuredFtsQuery(); +FtsQuery phraseFtsQuery = new FtsQuery(); + +PhraseQuery phrase = new PhraseQuery(); +phrase.setTerms("machine learning"); // Find exact phrase +phrase.setColumn("content"); +phrase.setSlop(1); // Allow 1 word between "machine" and "learning" +phraseFtsQuery.setPhrase(phrase); + +phraseStructured.setQuery(phraseFtsQuery); +phraseFullText.setStructuredQuery(phraseStructured); +phraseSearchQuery.setFullTextQuery(phraseFullText); + +byte[] phraseResults = namespace.queryTable(phraseSearchQuery); +// Expected: Documents with "machine learning" or "machine [word] learning" ``` -Other supported query types include: -- **Phrase Query**: Find exact phrases with optional slop (word distance) -- **Boost Query**: Boost documents matching certain criteria -- **Multi-Match Query**: Search across multiple fields with different weights - #### Hybrid Search Combining vector similarity search with full-text search often provides more relevant results than using either method alone. This is especially useful for semantic search applications where both conceptual similarity and keyword matching are important. ```java -// Hybrid search: vector + text +// Example: Find documents similar to a query embedding AND containing specific keywords QueryRequest hybridQuery = new QueryRequest(); -hybridQuery.setName("my_table"); - -// Vector search component -List queryVector = generateQueryVector(); // Your vector -hybridQuery.setVector(queryVector); +hybridQuery.setName("documents"); hybridQuery.setK(10); +hybridQuery.setColumns(Arrays.asList("id", "title", "content")); + +// Vector search component - find semantically similar documents +List queryEmbedding = new ArrayList<>(); +// In practice, this would be generated from a query text using an embedding model +for (int i = 0; i < 384; i++) { + queryEmbedding.add((float) Math.random()); +} +hybridQuery.setVector(queryEmbedding); -// Text search component +// Text search component - must also contain specific keywords QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); StringFtsQuery fts = new StringFtsQuery(); -fts.setQuery("search terms"); -fts.setColumns(Arrays.asList("content", "title")); // Optional: columns to search +fts.setQuery("neural networks"); // Require these keywords +fts.setColumns(Arrays.asList("content", "title")); fullTextQuery.setStringQuery(fts); hybridQuery.setFullTextQuery(fullTextQuery); -// Optional: Add filters -hybridQuery.setFilter("date > '2024-01-01'"); -hybridQuery.setPrefilter(true); -hybridQuery.setFastSearch(true); +// Optional: Add filter for recency +hybridQuery.setFilter("id > 2"); // Only recent documents +hybridQuery.setPrefilter(true); // Apply filter before search +hybridQuery.setFastSearch(true); // Use indexed data only byte[] hybridResults = namespace.queryTable(hybridQuery); +// Expected: Documents that are both semantically similar to the query +// AND contain "neural networks" keywords + +// Advanced Example: Hybrid search with structured FTS +QueryRequest advancedHybrid = new QueryRequest(); +advancedHybrid.setName("documents"); +advancedHybrid.setK(5); +advancedHybrid.setColumns(Arrays.asList("id", "title", "content")); + +// Vector component (same as above) +advancedHybrid.setVector(queryEmbedding); + +// Structured text search with boolean logic +QueryRequestFullTextQuery structuredFullText = new QueryRequestFullTextQuery(); +StructuredFtsQuery structured = new StructuredFtsQuery(); +FtsQuery structuredFts = new FtsQuery(); + +// Boolean: MUST have "learning" AND SHOULD have "deep" or "machine" +BooleanQuery hybridBool = new BooleanQuery(); + +FtsQuery mustHave = new FtsQuery(); +MatchQuery mustMatch = new MatchQuery(); +mustMatch.setTerms("learning"); +mustHave.setMatch(mustMatch); +hybridBool.setMust(Arrays.asList(mustHave)); + +// Add to query +structuredFts.setBoolean(hybridBool); +structured.setQuery(structuredFts); +structuredFullText.setStructuredQuery(structured); +advancedHybrid.setFullTextQuery(structuredFullText); + +byte[] advancedResults = namespace.queryTable(advancedHybrid); +// Expected: Semantically similar documents that contain "learning" ``` ### Creating a Vector Index @@ -578,28 +684,75 @@ if (!indexReady) { } ``` -### Updating and Deleting Data +### Inserting Additional Data + +```java +// Insert more rows into existing table +// Create Arrow data with same schema as original table +byte[] newData = createArrowData(/* new rows */); + +InsertTableResponse insertResponse = namespace.insertTable("my_table", newData); +System.out.println("Inserted " + insertResponse.getNumRows() + " new rows"); +``` + +### Counting Rows + +```java +CountRowsRequest countRequest = new CountRowsRequest(); +countRequest.setName("my_table"); + +long rowCount = namespace.countRows(countRequest); +System.out.println("Table has " + rowCount + " rows"); + +// Count with filter +countRequest.setFilter("id >= 100 AND id < 200"); +long filteredCount = namespace.countRows(countRequest); +System.out.println("Filtered count: " + filteredCount + " rows"); +``` + +### Updating Data ```java import com.lancedb.lance.namespace.model.UpdateTableRequest; -import com.lancedb.lance.namespace.model.DeleteFromTableRequest; -// Update rows +// Example: Update rows based on condition UpdateTableRequest updateRequest = new UpdateTableRequest(); updateRequest.setName("my_table"); -updateRequest.setPredicate("id > 100"); +updateRequest.setPredicate("id >= 50 AND id <= 60"); + List> updates = new ArrayList<>(); -updates.add(Arrays.asList("status", "'active'")); // Set status = 'active' +// Note: string values need quotes, numeric values don't +updates.add(Arrays.asList("some_field", "'updated_value'")); updateRequest.setUpdates(updates); UpdateTableResponse updateResponse = namespace.updateTable(updateRequest); +System.out.println("Updated " + updateResponse.getNumUpdatedRows() + " rows"); +``` -// Delete rows +### Deleting Data + +```java +import com.lancedb.lance.namespace.model.DeleteFromTableRequest; + +// Delete specific rows DeleteFromTableRequest deleteRequest = new DeleteFromTableRequest(); deleteRequest.setName("my_table"); -deleteRequest.setPredicate("status = 'inactive'"); +deleteRequest.setPredicate("id > 900"); DeleteFromTableResponse deleteResponse = namespace.deleteFromTable(deleteRequest); +System.out.println("Deleted " + deleteResponse.getNumDeletedRows() + " rows"); +``` + +### Describing a Table + +```java +DescribeTableRequest describeRequest = new DescribeTableRequest(); +describeRequest.setName("my_table"); + +DescribeTableResponse tableInfo = namespace.describeTable(describeRequest); +System.out.println("Table: " + tableInfo.getName()); +System.out.println("Schema: " + tableInfo.getSchema()); +System.out.println("Row count: " + tableInfo.getNumRows()); ``` ### Merge Insert (Upsert) From ad71b381ced275c8b227d836af5b0954b03079fb Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sun, 20 Jul 2025 11:39:44 -0700 Subject: [PATCH 06/10] Remove two mistakenly added doc --- CLAUDE.md | 135 ------------------------- java/LANCEDB_VECTOR_QUERY_ANALYSIS.md | 140 -------------------------- 2 files changed, 275 deletions(-) delete mode 100644 CLAUDE.md delete mode 100644 java/LANCEDB_VECTOR_QUERY_ANALYSIS.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index f6903d467..000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,135 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## OpenAPI oneOf Pattern Handling - -When working with OpenAPI specifications in this project, avoid using `oneOf` patterns as they cause compatibility issues across different language code generators. Instead, use the properties-based pattern similar to `AlterTransactionAction`: - -### Don't use oneOf: -```yaml -FtsQuery: - oneOf: - - type: object - required: [match] - properties: - match: - $ref: '#/components/schemas/MatchQuery' - - type: object - required: [phrase] - properties: - phrase: - $ref: '#/components/schemas/PhraseQuery' -``` - -### Do use properties pattern: -```yaml -FtsQuery: - type: object - description: | - Full-text search query. Exactly one query type field must be provided. - properties: - match: - $ref: '#/components/schemas/MatchQuery' - phrase: - $ref: '#/components/schemas/PhraseQuery' - boost: - $ref: '#/components/schemas/BoostQuery' -``` - -This pattern ensures better compatibility across different code generators (Java, Python, Rust) and maintains consistency with other parts of the API specification. - -## Project Overview - -Lance Namespace is an open specification for standardizing access to collections of Lance tables. The project provides: - -- An OpenAPI specification for REST-based namespace operations -- Multi-language client and server implementations (Java, Python, Rust) -- Generated code from the OpenAPI specification -- Documentation and examples - -## Architecture - -The project follows a specification-driven approach: - -1. **Core Specification**: OpenAPI spec at `docs/src/spec/rest.yaml` defines all operations -2. **Generated Code**: Client and server code is generated from the spec using `openapi-generator-cli` -3. **Multi-language Support**: Java, Python, and Rust implementations are provided -4. **Documentation**: MkDocs-based documentation in `docs/` directory - -### Key Components - -- **Java Core**: `java/lance-namespace-core/` - Core Java interface and utilities -- **Java Adapter**: `java/lance-namespace-adapter/` - Spring Boot server adapter -- **Generated Clients**: Auto-generated clients for each language -- **Generated Servers**: Auto-generated Spring Boot server stub - -## Development Commands - -### Project-wide Commands (from root) -```bash -# Lint the OpenAPI specification -make lint - -# Generate all clients and servers -make gen - -# Build all components -make build - -# Clean all generated code -make clean -``` - -### Language-specific Commands - -#### Java (from `java/` directory) -```bash -# Generate Java clients and servers -make gen - -# Build with Maven (includes linting with Spotless) -make build - -# Build specific components -make build-java-core -make build-java-adapter -``` - -#### Python (from `python/` directory) -```bash -# Generate Python client -make gen - -# Build and test with Poetry -make build -``` - -#### Rust (from `rust/` directory) -```bash -# Generate Rust client -make gen - -# Build and test with Cargo -make build -``` - -## Code Generation Workflow - -1. All client and server code is generated from `docs/src/spec/rest.yaml` -2. Generated code is placed in respective language directories -3. Build process includes generation, linting, and testing -4. Some generated files are cleaned up post-generation (metadata, git files) - -## Testing - -- **Java**: Tests run via Maven (`./mvnw install`) -- **Python**: Tests run via Poetry (`poetry run pytest`) -- **Rust**: Tests run via Cargo (`cargo test`) - -## Key Files - -- `docs/src/spec/rest.yaml`: OpenAPI specification (source of truth) -- `java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceNamespace.java`: Core Java interface -- `java/lance-namespace-adapter/`: Spring Boot server implementation -- Language-specific Makefiles for build automation \ No newline at end of file diff --git a/java/LANCEDB_VECTOR_QUERY_ANALYSIS.md b/java/LANCEDB_VECTOR_QUERY_ANALYSIS.md deleted file mode 100644 index e2d2cd5e5..000000000 --- a/java/LANCEDB_VECTOR_QUERY_ANALYSIS.md +++ /dev/null @@ -1,140 +0,0 @@ -# LanceDB Rust Vector Query Implementation Analysis - -## Overview - -This document analyzes how LanceDB's Rust implementation handles vector query requests, focusing on the QueryVector handling and oneOf type patterns. - -## Key Components - -### 1. Query Structure Hierarchy - -The Rust implementation uses an enum pattern for different query types: - -```rust -pub enum AnyQuery { - Query(QueryRequest), // Regular queries without vector search - VectorQuery(VectorQueryRequest), // Vector-based queries -} -``` - -### 2. VectorQueryRequest Structure - -```rust -pub struct VectorQueryRequest { - pub base: QueryRequest, // Base query parameters - pub column: Option, // Target vector column - pub query_vector: Vec>, // Query vectors (supports multiple) - pub minimum_nprobes: usize, - pub maximum_nprobes: Option, - pub lower_bound: Option, - pub upper_bound: Option, - pub distance_type: Option, - pub ef: Option, - pub refine_factor: Option, - pub use_index: bool, -} -``` - -### 3. Vector Handling - -#### Input Types -- Vectors are stored as `Vec>` allowing multiple query vectors -- The implementation supports various input types through the `IntoQueryVector` trait -- Common conversions: `Vec`, `&[f32]`, `&[f16]`, Arrow arrays - -#### JSON Serialization -When sending to the server, vectors are converted to JSON: - -```rust -fn vector_to_json(vector: &arrow_array::ArrayRef) -> Result { - match vector.data_type() { - DataType::Float32 => { - let array = vector.as_any().downcast_ref::().unwrap(); - Ok(serde_json::Value::Array( - array.values().iter() - .map(|v| serde_json::Value::Number( - serde_json::Number::from_f64(*v as f64).unwrap() - )) - .collect(), - )) - } - _ => Err(Error::InvalidInput { - message: "VectorQuery vector must be of type Float32".into(), - }), - } -} -``` - -### 4. Query Building Pattern - -The Rust implementation uses a builder pattern: - -```rust -table.query() - .nearest_to(&[1.0, 2.0, 3.0]) // Sets the first query vector - .add_query_vector(&[4.0, 5.0, 6.0]) // Adds additional vectors - .column("my_vector_column") - .distance_type(DistanceType::Cosine) - .limit(10) - .execute() -``` - -### 5. Server Request Format - -The implementation prepares different request formats based on the number of query vectors: - -1. **No vectors**: `"vector": []` -2. **Single vector**: `"vector": [1.0, 2.0, 3.0]` -3. **Multiple vectors** (if server supports): - - New format: `"vector": [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]` - - Old format: Sends multiple separate requests - -### 6. Key Insights for Java Implementation - -1. **OneOf Pattern**: The Rust code uses enum for Query/VectorQuery distinction, which maps to OpenAPI oneOf -2. **Vector Format**: Always Float32 arrays serialized as JSON number arrays -3. **Multiple Vectors**: Support for batch vector queries with query_index in results -4. **Builder Pattern**: Clean API for constructing complex queries -5. **Server Version Handling**: Different behavior based on server capabilities - -### 7. Request Body Structure - -A typical vector query request body: - -```json -{ - "version": null, // or specific version number - "vector": [0.1, 0.2, 0.3], - "vector_column": "embeddings", - "distance_type": "cosine", - "k": 10, - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 0, - "lower_bound": null, - "upper_bound": null, - "ef": null, - "refine_factor": null, - "bypass_vector_index": false, - "prefilter": true, - "filter": "id > 100", - "columns": ["id", "text", "metadata"] -} -``` - -### 8. Important Implementation Details - -1. **Type Safety**: Vectors must be Float32 type -2. **Empty Vector Handling**: Empty array `[]` means no vector search -3. **Column Auto-detection**: If column is not specified, server auto-detects -4. **Backward Compatibility**: Handles both old (nprobes) and new (minimum/maximum_nprobes) formats -5. **Timeout Support**: Query execution supports timeouts via headers - -## Recommendations for Java Implementation - -1. Use a similar builder pattern for query construction -2. Implement proper type conversion for vectors (List -> JSON array) -3. Handle the oneOf pattern with proper query type discrimination -4. Support both single and multiple vector queries -5. Implement server version detection for feature compatibility -6. Ensure proper JSON serialization matching the expected format \ No newline at end of file From f017b553cdac70b3caca74d7274dec3b002e38a4 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sun, 20 Jul 2025 12:03:57 -0700 Subject: [PATCH 07/10] fix test failure --- docs/src/user-guide/java-sdk.md | 36 ++++++++++++------- .../test/query/FullTextSearchTest.java | 10 ++++-- .../test/table/TableLifecycleTest.java | 3 +- .../lance/namespace/test/utils/TestUtils.java | 6 +++- 4 files changed, 38 insertions(+), 17 deletions(-) diff --git a/docs/src/user-guide/java-sdk.md b/docs/src/user-guide/java-sdk.md index 53f424ff9..5fd06565c 100644 --- a/docs/src/user-guide/java-sdk.md +++ b/docs/src/user-guide/java-sdk.md @@ -179,13 +179,11 @@ For more complex schemas (e.g., with text fields for full-text search, categoric Query results are returned in Arrow File format. Use `ArrowFileReader` to read the results. -!!! important "Column Selection Required" - When querying a table, you MUST specify which columns to return using `setColumns()`. If no columns are specified, the query will fail with an error: "no columns were selected and with_row_id is false, there is nothing to scan". - #### Vector Search ```java import com.lancedb.lance.namespace.model.QueryRequest; +import com.lancedb.lance.namespace.model.QueryRequestVector; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.vector.ipc.message.ArrowBlock; @@ -200,10 +198,14 @@ queryRequest.setName("my_vectors"); queryRequest.setK(10); // Get top 10 results // Create query vector (in practice, this would be your actual query embedding) -List queryVector = new ArrayList<>(); +List vectorList = new ArrayList<>(); for (int i = 0; i < 128; i++) { - queryVector.add((float) Math.random()); + vectorList.add((float) Math.random()); } + +// Wrap the vector in QueryRequestVector +QueryRequestVector queryVector = new QueryRequestVector(); +queryVector.setSingleVector(vectorList); queryRequest.setVector(queryVector); // REQUIRED: Specify columns to return @@ -266,10 +268,14 @@ vectorWithFilter.setName("my_vectors"); vectorWithFilter.setK(5); // Create query vector -List queryVector = new ArrayList<>(); +List vectorList = new ArrayList<>(); for (int i = 0; i < 128; i++) { - queryVector.add((float) Math.random()); + vectorList.add((float) Math.random()); } + +// Wrap the vector in QueryRequestVector +QueryRequestVector queryVector = new QueryRequestVector(); +queryVector.setSingleVector(vectorList); vectorWithFilter.setVector(queryVector); // Only search within specific ID range @@ -302,7 +308,7 @@ When combining vector search with filters, use `prefilter` to control the order // Prefiltering - filter first, then search vectors QueryRequest prefilterQuery = new QueryRequest(); prefilterQuery.setName("my_table"); -prefilterQuery.setVector(queryVector); +prefilterQuery.setVector(queryVector); // Assumes queryVector is a QueryRequestVector prefilterQuery.setK(10); prefilterQuery.setFilter("status = 'active'"); prefilterQuery.setPrefilter(true); @@ -311,7 +317,7 @@ prefilterQuery.setFastSearch(true); // Postfiltering - search vectors first, then filter (default) QueryRequest postfilterQuery = new QueryRequest(); postfilterQuery.setName("my_table"); -postfilterQuery.setVector(queryVector); +postfilterQuery.setVector(queryVector); // Assumes queryVector is a QueryRequestVector postfilterQuery.setK(10); postfilterQuery.setFilter("category = 'electronics'"); postfilterQuery.setPrefilter(false); @@ -478,7 +484,11 @@ List queryEmbedding = new ArrayList<>(); for (int i = 0; i < 384; i++) { queryEmbedding.add((float) Math.random()); } -hybridQuery.setVector(queryEmbedding); + +// Wrap the vector in QueryRequestVector +QueryRequestVector queryVector = new QueryRequestVector(); +queryVector.setSingleVector(queryEmbedding); +hybridQuery.setVector(queryVector); // Text search component - must also contain specific keywords QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); @@ -503,8 +513,10 @@ advancedHybrid.setName("documents"); advancedHybrid.setK(5); advancedHybrid.setColumns(Arrays.asList("id", "title", "content")); -// Vector component (same as above) -advancedHybrid.setVector(queryEmbedding); +// Vector component - wrap in QueryRequestVector +QueryRequestVector queryVector = new QueryRequestVector(); +queryVector.setSingleVector(queryEmbedding); +advancedHybrid.setVector(queryVector); // Structured text search with boolean logic QueryRequestFullTextQuery structuredFullText = new QueryRequestFullTextQuery(); diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java index ba4162a2c..ccf02be9d 100644 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java @@ -391,10 +391,14 @@ public void testHybridSearch() throws IOException, InterruptedException { hybridQuery.setColumns(Arrays.asList("id", "text")); // Add vector search - search for vector with all 10s - List queryVector = new ArrayList<>(); + List vectorList = new ArrayList<>(); for (int i = 0; i < 128; i++) { - queryVector.add(10.0f); + vectorList.add(10.0f); } + + // Wrap the vector in QueryRequestVector + QueryRequestVector queryVector = new QueryRequestVector(); + queryVector.setSingleVector(vectorList); hybridQuery.setVector(queryVector); hybridQuery.setK(10); @@ -426,7 +430,7 @@ public void testHybridSearch() throws IOException, InterruptedException { System.out.println("\n--- Testing hybrid search with filter (id <= 10) ---"); QueryRequest hybridFilterQuery = new QueryRequest(); hybridFilterQuery.setName(tableName); - hybridFilterQuery.setVector(queryVector); + hybridFilterQuery.setVector(queryVector); // Reuse the same QueryRequestVector from above hybridFilterQuery.setK(20); hybridFilterQuery.setFilter("id <= 10"); hybridFilterQuery.setFullTextQuery(fullTextQuery); diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableLifecycleTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableLifecycleTest.java index 3eb1a4832..8ec9754f6 100644 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableLifecycleTest.java +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/table/TableLifecycleTest.java @@ -67,12 +67,13 @@ public void testTableLifecycle() throws IOException { JsonSchema responseSchema = describeResponse.getSchema(); assertNotNull(responseSchema, "Schema object should not be null"); assertNotNull(responseSchema.getFields(), "Schema fields should not be null"); - assertEquals(3, responseSchema.getFields().size(), "Schema should have 3 fields"); + assertEquals(4, responseSchema.getFields().size(), "Schema should have 4 fields"); List fieldNames = responseSchema.getFields().stream().map(JsonField::getName).collect(Collectors.toList()); assertTrue(fieldNames.contains("id"), "Schema should contain 'id' field"); assertTrue(fieldNames.contains("name"), "Schema should contain 'name' field"); + assertTrue(fieldNames.contains("category"), "Schema should contain 'category' field"); assertTrue(fieldNames.contains("embedding"), "Schema should contain 'embedding' field"); System.out.println("✓ Table schema verified with fields: " + fieldNames); diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java index f7cdba44f..66a959887 100644 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java @@ -134,7 +134,11 @@ public static QueryRequest createVectorQuery( for (int i = 0; i < dimensions; i++) { vector.add(targetValue); } - query.setVector(vector); + + // Wrap the vector in QueryRequestVector + QueryRequestVector queryVector = new QueryRequestVector(); + queryVector.setSingleVector(vector); + query.setVector(queryVector); // Add default columns to avoid "no columns selected" error query.setColumns(java.util.Arrays.asList("id", "name", "category", "embedding")); From a3dfbe8630398a68b29cfbfd292e62f429e9f893 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sun, 20 Jul 2025 14:48:32 -0700 Subject: [PATCH 08/10] tmp --- .../com/lancedb/lance/namespace/test/index/IndexTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java index acdb51db0..53ae358d9 100644 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java @@ -193,7 +193,7 @@ public void testMultipleIndices() throws IOException, InterruptedException { try { // Create table with text field System.out.println("\n--- Creating table with multiple fields ---"); - byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 100).build(); + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 300).build(); namespace.createTable(tableName, tableData); // Create vector index @@ -265,7 +265,7 @@ public void testIndexWithDifferentMetrics() throws IOException, InterruptedExcep try { // Create table - byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 200).build(); + byte[] tableData = new ArrowTestUtils.TableDataBuilder(allocator).addRows(1, 300).build(); namespace.createTable(tableName, tableData); // Test COSINE metric From 051588ec14241c6bd002d50b19b5f194eb922837 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sun, 20 Jul 2025 16:07:52 -0700 Subject: [PATCH 09/10] Fix oneof issue and serde utagged issues --- docs/src/spec/rest.yaml | 56 +-- docs/src/user-guide/java-sdk.md | 113 ++---- .../api/openapi.yaml | 71 ++-- .../docs/CreateIndexRequest.md | 13 +- .../docs/MatchQuery.md | 2 +- .../namespace/model/CreateIndexRequest.java | 341 +++++++++++++----- .../lance/namespace/model/MatchQuery.java | 30 +- .../lance/namespace/LanceRestNamespace.java | 8 + .../jackson/LanceNamespaceJacksonModule.java | 29 ++ .../jackson/QueryRequestSerializer.java | 146 ++++++++ .../lance/namespace/test/index/IndexTest.java | 8 +- .../test/jackson/DebugSerializerTest.java | 69 ++++ .../jackson/QueryRequestSerializerTest.java | 157 ++++++++ .../test/query/FullTextSearchTest.java | 129 +------ .../lance/namespace/test/utils/TestUtils.java | 2 - .../springboot/model/CreateIndexRequest.java | 241 ++++++++----- .../server/springboot/model/MatchQuery.java | 6 +- .../docs/CreateIndexRequest.md | 13 +- .../docs/MatchQuery.md | 2 +- .../models/create_index_request.py | 30 +- .../models/match_query.py | 2 +- .../test/test_create_index_request.py | 13 +- .../test/test_match_query.py | 1 + .../docs/CreateIndexRequest.md | 13 +- .../docs/MatchQuery.md | 2 +- .../src/models/create_index_request.rs | 52 ++- .../src/models/match_query.rs | 8 +- 27 files changed, 1038 insertions(+), 519 deletions(-) create mode 100644 java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/jackson/LanceNamespaceJacksonModule.java create mode 100644 java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/jackson/QueryRequestSerializer.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/jackson/DebugSerializerTest.java create mode 100644 java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/jackson/QueryRequestSerializerTest.java diff --git a/docs/src/spec/rest.yaml b/docs/src/spec/rest.yaml index bcc4aad84..234b5b006 100644 --- a/docs/src/spec/rest.yaml +++ b/docs/src/spec/rest.yaml @@ -1639,29 +1639,42 @@ components: metric_type: type: string enum: ["l2", "cosine", "dot"] + nullable: true description: Distance metric type for vector indexes - num_partitions: - type: integer - minimum: 1 - description: Number of partitions for IVF indexes - num_sub_vectors: - type: integer - minimum: 1 - description: Number of sub-vectors for PQ indexes - num_bits: - type: integer - minimum: 1 - maximum: 8 - description: Number of bits for scalar quantization - max_iterations: - type: integer - minimum: 1 - description: Maximum iterations for index building - sample_rate: + with_position: + type: boolean + nullable: true + description: Optional FTS parameter for position tracking + base_tokenizer: + type: string + nullable: true + description: Optional FTS parameter for base tokenizer + language: + type: string + nullable: true + description: Optional FTS parameter for language + max_token_length: type: integer - minimum: 1 - description: Sample rate for index building - + nullable: true + minimum: 0 + description: Optional FTS parameter for maximum token length + lower_case: + type: boolean + nullable: true + description: Optional FTS parameter for lowercase conversion + stem: + type: boolean + nullable: true + description: Optional FTS parameter for stemming + remove_stop_words: + type: boolean + nullable: true + description: Optional FTS parameter for stop word removal + ascii_folding: + type: boolean + nullable: true + description: Optional FTS parameter for ASCII folding + CreateIndexResponse: type: object required: @@ -2148,6 +2161,7 @@ components: type: object required: - terms + - column properties: boost: type: number diff --git a/docs/src/user-guide/java-sdk.md b/docs/src/user-guide/java-sdk.md index 5fd06565c..6d757709e 100644 --- a/docs/src/user-guide/java-sdk.md +++ b/docs/src/user-guide/java-sdk.md @@ -183,7 +183,6 @@ Query results are returned in Arrow File format. Use `ArrowFileReader` to read t ```java import com.lancedb.lance.namespace.model.QueryRequest; -import com.lancedb.lance.namespace.model.QueryRequestVector; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.vector.ipc.message.ArrowBlock; @@ -198,14 +197,10 @@ queryRequest.setName("my_vectors"); queryRequest.setK(10); // Get top 10 results // Create query vector (in practice, this would be your actual query embedding) -List vectorList = new ArrayList<>(); +List queryVector = new ArrayList<>(); for (int i = 0; i < 128; i++) { - vectorList.add((float) Math.random()); + queryVector.add((float) Math.random()); } - -// Wrap the vector in QueryRequestVector -QueryRequestVector queryVector = new QueryRequestVector(); -queryVector.setSingleVector(vectorList); queryRequest.setVector(queryVector); // REQUIRED: Specify columns to return @@ -268,14 +263,10 @@ vectorWithFilter.setName("my_vectors"); vectorWithFilter.setK(5); // Create query vector -List vectorList = new ArrayList<>(); +List queryVector = new ArrayList<>(); for (int i = 0; i < 128; i++) { - vectorList.add((float) Math.random()); + queryVector.add((float) Math.random()); } - -// Wrap the vector in QueryRequestVector -QueryRequestVector queryVector = new QueryRequestVector(); -queryVector.setSingleVector(vectorList); vectorWithFilter.setVector(queryVector); // Only search within specific ID range @@ -308,7 +299,7 @@ When combining vector search with filters, use `prefilter` to control the order // Prefiltering - filter first, then search vectors QueryRequest prefilterQuery = new QueryRequest(); prefilterQuery.setName("my_table"); -prefilterQuery.setVector(queryVector); // Assumes queryVector is a QueryRequestVector +prefilterQuery.setVector(queryVector); prefilterQuery.setK(10); prefilterQuery.setFilter("status = 'active'"); prefilterQuery.setPrefilter(true); @@ -317,7 +308,7 @@ prefilterQuery.setFastSearch(true); // Postfiltering - search vectors first, then filter (default) QueryRequest postfilterQuery = new QueryRequest(); postfilterQuery.setName("my_table"); -postfilterQuery.setVector(queryVector); // Assumes queryVector is a QueryRequestVector +postfilterQuery.setVector(queryVector); postfilterQuery.setK(10); postfilterQuery.setFilter("category = 'electronics'"); postfilterQuery.setPrefilter(false); @@ -336,6 +327,8 @@ CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); ftsIndexRequest.setName("documents"); ftsIndexRequest.setColumn("content"); ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); +// Note: Set withPosition=true if you plan to use PhraseQuery +// ftsIndexRequest.setWithPosition(true); CreateIndexResponse ftsResponse = namespace.createIndex(ftsIndexRequest); // Wait for index to be built @@ -443,6 +436,7 @@ byte[] boolResults = namespace.queryTable(booleanSearchQuery); // Expected: Documents containing "learning" (required) and preferably "machine" or "deep" // Example 2: Phrase Query - Find exact phrases +// IMPORTANT: PhraseQuery requires the FTS index to be created with withPosition=true QueryRequest phraseSearchQuery = new QueryRequest(); phraseSearchQuery.setName("documents"); phraseSearchQuery.setK(5); @@ -467,80 +461,17 @@ byte[] phraseResults = namespace.queryTable(phraseSearchQuery); // Expected: Documents with "machine learning" or "machine [word] learning" ``` -#### Hybrid Search - -Combining vector similarity search with full-text search often provides more relevant results than using either method alone. This is especially useful for semantic search applications where both conceptual similarity and keyword matching are important. - -```java -// Example: Find documents similar to a query embedding AND containing specific keywords -QueryRequest hybridQuery = new QueryRequest(); -hybridQuery.setName("documents"); -hybridQuery.setK(10); -hybridQuery.setColumns(Arrays.asList("id", "title", "content")); - -// Vector search component - find semantically similar documents -List queryEmbedding = new ArrayList<>(); -// In practice, this would be generated from a query text using an embedding model -for (int i = 0; i < 384; i++) { - queryEmbedding.add((float) Math.random()); -} - -// Wrap the vector in QueryRequestVector -QueryRequestVector queryVector = new QueryRequestVector(); -queryVector.setSingleVector(queryEmbedding); -hybridQuery.setVector(queryVector); - -// Text search component - must also contain specific keywords -QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); -StringFtsQuery fts = new StringFtsQuery(); -fts.setQuery("neural networks"); // Require these keywords -fts.setColumns(Arrays.asList("content", "title")); -fullTextQuery.setStringQuery(fts); -hybridQuery.setFullTextQuery(fullTextQuery); - -// Optional: Add filter for recency -hybridQuery.setFilter("id > 2"); // Only recent documents -hybridQuery.setPrefilter(true); // Apply filter before search -hybridQuery.setFastSearch(true); // Use indexed data only - -byte[] hybridResults = namespace.queryTable(hybridQuery); -// Expected: Documents that are both semantically similar to the query -// AND contain "neural networks" keywords - -// Advanced Example: Hybrid search with structured FTS -QueryRequest advancedHybrid = new QueryRequest(); -advancedHybrid.setName("documents"); -advancedHybrid.setK(5); -advancedHybrid.setColumns(Arrays.asList("id", "title", "content")); - -// Vector component - wrap in QueryRequestVector -QueryRequestVector queryVector = new QueryRequestVector(); -queryVector.setSingleVector(queryEmbedding); -advancedHybrid.setVector(queryVector); - -// Structured text search with boolean logic -QueryRequestFullTextQuery structuredFullText = new QueryRequestFullTextQuery(); -StructuredFtsQuery structured = new StructuredFtsQuery(); -FtsQuery structuredFts = new FtsQuery(); - -// Boolean: MUST have "learning" AND SHOULD have "deep" or "machine" -BooleanQuery hybridBool = new BooleanQuery(); - -FtsQuery mustHave = new FtsQuery(); -MatchQuery mustMatch = new MatchQuery(); -mustMatch.setTerms("learning"); -mustHave.setMatch(mustMatch); -hybridBool.setMust(Arrays.asList(mustHave)); - -// Add to query -structuredFts.setBoolean(hybridBool); -structured.setQuery(structuredFts); -structuredFullText.setStructuredQuery(structured); -advancedHybrid.setFullTextQuery(structuredFullText); - -byte[] advancedResults = namespace.queryTable(advancedHybrid); -// Expected: Semantically similar documents that contain "learning" -``` +!!! warning "PhraseQuery Requirements" + PhraseQuery requires the FTS index to be created with `withPosition=true`. If you attempt to use PhraseQuery on an index created without position information, you will receive an error: "position is not found but required for phrase queries". + + Always create your FTS index with position enabled if you plan to use phrase searches: + ```java + CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); + ftsIndexRequest.setName("documents"); + ftsIndexRequest.setColumn("content"); + ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); + ftsIndexRequest.setWithPosition(true); // Required for PhraseQuery + ``` ### Creating a Vector Index @@ -792,6 +723,10 @@ System.out.println("Updated rows: " + response.getNumUpdatedRows()); System.out.println("Inserted rows: " + response.getNumInsertedRows()); ``` +## Known Limitation +### Not Supported: Hybrid Search +Hybrid Search requires a vector search and a full text search, cannot run both in one query. +Need higher level of search orchestration to provide user level hybrid search operations. ## Additional Resources diff --git a/java/lance-namespace-apache-client/api/openapi.yaml b/java/lance-namespace-apache-client/api/openapi.yaml index fcdddde97..1f221c0d6 100644 --- a/java/lance-namespace-apache-client/api/openapi.yaml +++ b/java/lance-namespace-apache-client/api/openapi.yaml @@ -2188,18 +2188,21 @@ components: - vector CreateIndexRequest: example: - max_iterations: 1 - num_sub_vectors: 1 - sample_rate: 1 + base_tokenizer: base_tokenizer + column: column + max_token_length: 0 + language: language + index_type: BTREE + with_position: true + lower_case: true name: name namespace: - namespace - namespace - column: column metric_type: l2 - num_partitions: 1 - num_bits: 2 - index_type: BTREE + ascii_folding: true + remove_stop_words: true + stem: true properties: name: description: The table name @@ -2230,27 +2233,40 @@ components: - cosine - dot type: string - num_partitions: - description: Number of partitions for IVF indexes - minimum: 1 - type: integer - num_sub_vectors: - description: Number of sub-vectors for PQ indexes - minimum: 1 - type: integer - num_bits: - description: Number of bits for scalar quantization - maximum: 8 - minimum: 1 - type: integer - max_iterations: - description: Maximum iterations for index building - minimum: 1 - type: integer - sample_rate: - description: Sample rate for index building - minimum: 1 + nullable: true + with_position: + description: Optional FTS parameter for position tracking + type: boolean + nullable: true + base_tokenizer: + description: Optional FTS parameter for base tokenizer + type: string + nullable: true + language: + description: Optional FTS parameter for language + type: string + nullable: true + max_token_length: + description: Optional FTS parameter for maximum token length + minimum: 0 type: integer + nullable: true + lower_case: + description: Optional FTS parameter for lowercase conversion + type: boolean + nullable: true + stem: + description: Optional FTS parameter for stemming + type: boolean + nullable: true + remove_stop_words: + description: Optional FTS parameter for stop word removal + type: boolean + nullable: true + ascii_folding: + description: Optional FTS parameter for ASCII folding + type: boolean + nullable: true required: - column - index_type @@ -2972,6 +2988,7 @@ components: terms: type: string required: + - column - terms PhraseQuery: example: diff --git a/java/lance-namespace-apache-client/docs/CreateIndexRequest.md b/java/lance-namespace-apache-client/docs/CreateIndexRequest.md index ca507bc82..85dc3a19e 100644 --- a/java/lance-namespace-apache-client/docs/CreateIndexRequest.md +++ b/java/lance-namespace-apache-client/docs/CreateIndexRequest.md @@ -12,11 +12,14 @@ |**column** | **String** | Name of the column to create index on | | |**indexType** | [**IndexTypeEnum**](#IndexTypeEnum) | Type of index to create | | |**metricType** | [**MetricTypeEnum**](#MetricTypeEnum) | Distance metric type for vector indexes | [optional] | -|**numPartitions** | **Integer** | Number of partitions for IVF indexes | [optional] | -|**numSubVectors** | **Integer** | Number of sub-vectors for PQ indexes | [optional] | -|**numBits** | **Integer** | Number of bits for scalar quantization | [optional] | -|**maxIterations** | **Integer** | Maximum iterations for index building | [optional] | -|**sampleRate** | **Integer** | Sample rate for index building | [optional] | +|**withPosition** | **Boolean** | Optional FTS parameter for position tracking | [optional] | +|**baseTokenizer** | **String** | Optional FTS parameter for base tokenizer | [optional] | +|**language** | **String** | Optional FTS parameter for language | [optional] | +|**maxTokenLength** | **Integer** | Optional FTS parameter for maximum token length | [optional] | +|**lowerCase** | **Boolean** | Optional FTS parameter for lowercase conversion | [optional] | +|**stem** | **Boolean** | Optional FTS parameter for stemming | [optional] | +|**removeStopWords** | **Boolean** | Optional FTS parameter for stop word removal | [optional] | +|**asciiFolding** | **Boolean** | Optional FTS parameter for ASCII folding | [optional] | diff --git a/java/lance-namespace-apache-client/docs/MatchQuery.md b/java/lance-namespace-apache-client/docs/MatchQuery.md index 2a123367f..02e4b129d 100644 --- a/java/lance-namespace-apache-client/docs/MatchQuery.md +++ b/java/lance-namespace-apache-client/docs/MatchQuery.md @@ -8,7 +8,7 @@ | Name | Type | Description | Notes | |------------ | ------------- | ------------- | -------------| |**boost** | **Float** | | [optional] | -|**column** | **String** | | [optional] | +|**column** | **String** | | | |**fuzziness** | **Integer** | | [optional] | |**maxExpansions** | **Integer** | The maximum number of terms to expand for fuzzy matching. Default to 50. | [optional] | |**operator** | **Operator** | The operator to use for combining terms. This can be either `And` or `Or`, it's 'Or' by default. - `And`: All terms must match. - `Or`: At least one term must match. | [optional] | diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/CreateIndexRequest.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/CreateIndexRequest.java index 0cd69f4b2..3156456f0 100644 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/CreateIndexRequest.java +++ b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/CreateIndexRequest.java @@ -33,11 +33,14 @@ CreateIndexRequest.JSON_PROPERTY_COLUMN, CreateIndexRequest.JSON_PROPERTY_INDEX_TYPE, CreateIndexRequest.JSON_PROPERTY_METRIC_TYPE, - CreateIndexRequest.JSON_PROPERTY_NUM_PARTITIONS, - CreateIndexRequest.JSON_PROPERTY_NUM_SUB_VECTORS, - CreateIndexRequest.JSON_PROPERTY_NUM_BITS, - CreateIndexRequest.JSON_PROPERTY_MAX_ITERATIONS, - CreateIndexRequest.JSON_PROPERTY_SAMPLE_RATE + CreateIndexRequest.JSON_PROPERTY_WITH_POSITION, + CreateIndexRequest.JSON_PROPERTY_BASE_TOKENIZER, + CreateIndexRequest.JSON_PROPERTY_LANGUAGE, + CreateIndexRequest.JSON_PROPERTY_MAX_TOKEN_LENGTH, + CreateIndexRequest.JSON_PROPERTY_LOWER_CASE, + CreateIndexRequest.JSON_PROPERTY_STEM, + CreateIndexRequest.JSON_PROPERTY_REMOVE_STOP_WORDS, + CreateIndexRequest.JSON_PROPERTY_ASCII_FOLDING }) @javax.annotation.Generated( value = "org.openapitools.codegen.languages.JavaClientCodegen", @@ -136,20 +139,29 @@ public static MetricTypeEnum fromValue(String value) { public static final String JSON_PROPERTY_METRIC_TYPE = "metric_type"; @javax.annotation.Nullable private MetricTypeEnum metricType; - public static final String JSON_PROPERTY_NUM_PARTITIONS = "num_partitions"; - @javax.annotation.Nullable private Integer numPartitions; + public static final String JSON_PROPERTY_WITH_POSITION = "with_position"; + @javax.annotation.Nullable private Boolean withPosition; - public static final String JSON_PROPERTY_NUM_SUB_VECTORS = "num_sub_vectors"; - @javax.annotation.Nullable private Integer numSubVectors; + public static final String JSON_PROPERTY_BASE_TOKENIZER = "base_tokenizer"; + @javax.annotation.Nullable private String baseTokenizer; - public static final String JSON_PROPERTY_NUM_BITS = "num_bits"; - @javax.annotation.Nullable private Integer numBits; + public static final String JSON_PROPERTY_LANGUAGE = "language"; + @javax.annotation.Nullable private String language; - public static final String JSON_PROPERTY_MAX_ITERATIONS = "max_iterations"; - @javax.annotation.Nullable private Integer maxIterations; + public static final String JSON_PROPERTY_MAX_TOKEN_LENGTH = "max_token_length"; + @javax.annotation.Nullable private Integer maxTokenLength; - public static final String JSON_PROPERTY_SAMPLE_RATE = "sample_rate"; - @javax.annotation.Nullable private Integer sampleRate; + public static final String JSON_PROPERTY_LOWER_CASE = "lower_case"; + @javax.annotation.Nullable private Boolean lowerCase; + + public static final String JSON_PROPERTY_STEM = "stem"; + @javax.annotation.Nullable private Boolean stem; + + public static final String JSON_PROPERTY_REMOVE_STOP_WORDS = "remove_stop_words"; + @javax.annotation.Nullable private Boolean removeStopWords; + + public static final String JSON_PROPERTY_ASCII_FOLDING = "ascii_folding"; + @javax.annotation.Nullable private Boolean asciiFolding; public CreateIndexRequest() {} @@ -281,124 +293,196 @@ public void setMetricType(@javax.annotation.Nullable MetricTypeEnum metricType) this.metricType = metricType; } - public CreateIndexRequest numPartitions(@javax.annotation.Nullable Integer numPartitions) { + public CreateIndexRequest withPosition(@javax.annotation.Nullable Boolean withPosition) { - this.numPartitions = numPartitions; + this.withPosition = withPosition; return this; } /** - * Number of partitions for IVF indexes minimum: 1 + * Optional FTS parameter for position tracking * - * @return numPartitions + * @return withPosition */ @javax.annotation.Nullable - @JsonProperty(JSON_PROPERTY_NUM_PARTITIONS) + @JsonProperty(JSON_PROPERTY_WITH_POSITION) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public Integer getNumPartitions() { - return numPartitions; + public Boolean getWithPosition() { + return withPosition; } - @JsonProperty(JSON_PROPERTY_NUM_PARTITIONS) + @JsonProperty(JSON_PROPERTY_WITH_POSITION) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public void setNumPartitions(@javax.annotation.Nullable Integer numPartitions) { - this.numPartitions = numPartitions; + public void setWithPosition(@javax.annotation.Nullable Boolean withPosition) { + this.withPosition = withPosition; } - public CreateIndexRequest numSubVectors(@javax.annotation.Nullable Integer numSubVectors) { + public CreateIndexRequest baseTokenizer(@javax.annotation.Nullable String baseTokenizer) { - this.numSubVectors = numSubVectors; + this.baseTokenizer = baseTokenizer; return this; } /** - * Number of sub-vectors for PQ indexes minimum: 1 + * Optional FTS parameter for base tokenizer * - * @return numSubVectors + * @return baseTokenizer */ @javax.annotation.Nullable - @JsonProperty(JSON_PROPERTY_NUM_SUB_VECTORS) + @JsonProperty(JSON_PROPERTY_BASE_TOKENIZER) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public Integer getNumSubVectors() { - return numSubVectors; + public String getBaseTokenizer() { + return baseTokenizer; } - @JsonProperty(JSON_PROPERTY_NUM_SUB_VECTORS) + @JsonProperty(JSON_PROPERTY_BASE_TOKENIZER) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public void setNumSubVectors(@javax.annotation.Nullable Integer numSubVectors) { - this.numSubVectors = numSubVectors; + public void setBaseTokenizer(@javax.annotation.Nullable String baseTokenizer) { + this.baseTokenizer = baseTokenizer; } - public CreateIndexRequest numBits(@javax.annotation.Nullable Integer numBits) { + public CreateIndexRequest language(@javax.annotation.Nullable String language) { - this.numBits = numBits; + this.language = language; return this; } /** - * Number of bits for scalar quantization minimum: 1 maximum: 8 + * Optional FTS parameter for language * - * @return numBits + * @return language */ @javax.annotation.Nullable - @JsonProperty(JSON_PROPERTY_NUM_BITS) + @JsonProperty(JSON_PROPERTY_LANGUAGE) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public Integer getNumBits() { - return numBits; + public String getLanguage() { + return language; } - @JsonProperty(JSON_PROPERTY_NUM_BITS) + @JsonProperty(JSON_PROPERTY_LANGUAGE) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public void setNumBits(@javax.annotation.Nullable Integer numBits) { - this.numBits = numBits; + public void setLanguage(@javax.annotation.Nullable String language) { + this.language = language; } - public CreateIndexRequest maxIterations(@javax.annotation.Nullable Integer maxIterations) { + public CreateIndexRequest maxTokenLength(@javax.annotation.Nullable Integer maxTokenLength) { - this.maxIterations = maxIterations; + this.maxTokenLength = maxTokenLength; return this; } /** - * Maximum iterations for index building minimum: 1 + * Optional FTS parameter for maximum token length minimum: 0 * - * @return maxIterations + * @return maxTokenLength */ @javax.annotation.Nullable - @JsonProperty(JSON_PROPERTY_MAX_ITERATIONS) + @JsonProperty(JSON_PROPERTY_MAX_TOKEN_LENGTH) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public Integer getMaxIterations() { - return maxIterations; + public Integer getMaxTokenLength() { + return maxTokenLength; } - @JsonProperty(JSON_PROPERTY_MAX_ITERATIONS) + @JsonProperty(JSON_PROPERTY_MAX_TOKEN_LENGTH) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public void setMaxIterations(@javax.annotation.Nullable Integer maxIterations) { - this.maxIterations = maxIterations; + public void setMaxTokenLength(@javax.annotation.Nullable Integer maxTokenLength) { + this.maxTokenLength = maxTokenLength; } - public CreateIndexRequest sampleRate(@javax.annotation.Nullable Integer sampleRate) { + public CreateIndexRequest lowerCase(@javax.annotation.Nullable Boolean lowerCase) { - this.sampleRate = sampleRate; + this.lowerCase = lowerCase; return this; } /** - * Sample rate for index building minimum: 1 + * Optional FTS parameter for lowercase conversion * - * @return sampleRate + * @return lowerCase */ @javax.annotation.Nullable - @JsonProperty(JSON_PROPERTY_SAMPLE_RATE) + @JsonProperty(JSON_PROPERTY_LOWER_CASE) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public Integer getSampleRate() { - return sampleRate; + public Boolean getLowerCase() { + return lowerCase; } - @JsonProperty(JSON_PROPERTY_SAMPLE_RATE) + @JsonProperty(JSON_PROPERTY_LOWER_CASE) @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public void setSampleRate(@javax.annotation.Nullable Integer sampleRate) { - this.sampleRate = sampleRate; + public void setLowerCase(@javax.annotation.Nullable Boolean lowerCase) { + this.lowerCase = lowerCase; + } + + public CreateIndexRequest stem(@javax.annotation.Nullable Boolean stem) { + + this.stem = stem; + return this; + } + + /** + * Optional FTS parameter for stemming + * + * @return stem + */ + @javax.annotation.Nullable + @JsonProperty(JSON_PROPERTY_STEM) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public Boolean getStem() { + return stem; + } + + @JsonProperty(JSON_PROPERTY_STEM) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setStem(@javax.annotation.Nullable Boolean stem) { + this.stem = stem; + } + + public CreateIndexRequest removeStopWords(@javax.annotation.Nullable Boolean removeStopWords) { + + this.removeStopWords = removeStopWords; + return this; + } + + /** + * Optional FTS parameter for stop word removal + * + * @return removeStopWords + */ + @javax.annotation.Nullable + @JsonProperty(JSON_PROPERTY_REMOVE_STOP_WORDS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public Boolean getRemoveStopWords() { + return removeStopWords; + } + + @JsonProperty(JSON_PROPERTY_REMOVE_STOP_WORDS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setRemoveStopWords(@javax.annotation.Nullable Boolean removeStopWords) { + this.removeStopWords = removeStopWords; + } + + public CreateIndexRequest asciiFolding(@javax.annotation.Nullable Boolean asciiFolding) { + + this.asciiFolding = asciiFolding; + return this; + } + + /** + * Optional FTS parameter for ASCII folding + * + * @return asciiFolding + */ + @javax.annotation.Nullable + @JsonProperty(JSON_PROPERTY_ASCII_FOLDING) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public Boolean getAsciiFolding() { + return asciiFolding; + } + + @JsonProperty(JSON_PROPERTY_ASCII_FOLDING) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public void setAsciiFolding(@javax.annotation.Nullable Boolean asciiFolding) { + this.asciiFolding = asciiFolding; } @Override @@ -415,11 +499,14 @@ public boolean equals(Object o) { && Objects.equals(this.column, createIndexRequest.column) && Objects.equals(this.indexType, createIndexRequest.indexType) && Objects.equals(this.metricType, createIndexRequest.metricType) - && Objects.equals(this.numPartitions, createIndexRequest.numPartitions) - && Objects.equals(this.numSubVectors, createIndexRequest.numSubVectors) - && Objects.equals(this.numBits, createIndexRequest.numBits) - && Objects.equals(this.maxIterations, createIndexRequest.maxIterations) - && Objects.equals(this.sampleRate, createIndexRequest.sampleRate); + && Objects.equals(this.withPosition, createIndexRequest.withPosition) + && Objects.equals(this.baseTokenizer, createIndexRequest.baseTokenizer) + && Objects.equals(this.language, createIndexRequest.language) + && Objects.equals(this.maxTokenLength, createIndexRequest.maxTokenLength) + && Objects.equals(this.lowerCase, createIndexRequest.lowerCase) + && Objects.equals(this.stem, createIndexRequest.stem) + && Objects.equals(this.removeStopWords, createIndexRequest.removeStopWords) + && Objects.equals(this.asciiFolding, createIndexRequest.asciiFolding); } @Override @@ -430,11 +517,14 @@ public int hashCode() { column, indexType, metricType, - numPartitions, - numSubVectors, - numBits, - maxIterations, - sampleRate); + withPosition, + baseTokenizer, + language, + maxTokenLength, + lowerCase, + stem, + removeStopWords, + asciiFolding); } @Override @@ -446,11 +536,14 @@ public String toString() { sb.append(" column: ").append(toIndentedString(column)).append("\n"); sb.append(" indexType: ").append(toIndentedString(indexType)).append("\n"); sb.append(" metricType: ").append(toIndentedString(metricType)).append("\n"); - sb.append(" numPartitions: ").append(toIndentedString(numPartitions)).append("\n"); - sb.append(" numSubVectors: ").append(toIndentedString(numSubVectors)).append("\n"); - sb.append(" numBits: ").append(toIndentedString(numBits)).append("\n"); - sb.append(" maxIterations: ").append(toIndentedString(maxIterations)).append("\n"); - sb.append(" sampleRate: ").append(toIndentedString(sampleRate)).append("\n"); + sb.append(" withPosition: ").append(toIndentedString(withPosition)).append("\n"); + sb.append(" baseTokenizer: ").append(toIndentedString(baseTokenizer)).append("\n"); + sb.append(" language: ").append(toIndentedString(language)).append("\n"); + sb.append(" maxTokenLength: ").append(toIndentedString(maxTokenLength)).append("\n"); + sb.append(" lowerCase: ").append(toIndentedString(lowerCase)).append("\n"); + sb.append(" stem: ").append(toIndentedString(stem)).append("\n"); + sb.append(" removeStopWords: ").append(toIndentedString(removeStopWords)).append("\n"); + sb.append(" asciiFolding: ").append(toIndentedString(asciiFolding)).append("\n"); sb.append("}"); return sb.toString(); } @@ -580,15 +673,63 @@ public String toUrlQueryString(String prefix) { } } - // add `num_partitions` to the URL query string - if (getNumPartitions() != null) { + // add `with_position` to the URL query string + if (getWithPosition() != null) { + try { + joiner.add( + String.format( + "%swith_position%s=%s", + prefix, + suffix, + URLEncoder.encode(String.valueOf(getWithPosition()), "UTF-8") + .replaceAll("\\+", "%20"))); + } catch (UnsupportedEncodingException e) { + // Should never happen, UTF-8 is always supported + throw new RuntimeException(e); + } + } + + // add `base_tokenizer` to the URL query string + if (getBaseTokenizer() != null) { + try { + joiner.add( + String.format( + "%sbase_tokenizer%s=%s", + prefix, + suffix, + URLEncoder.encode(String.valueOf(getBaseTokenizer()), "UTF-8") + .replaceAll("\\+", "%20"))); + } catch (UnsupportedEncodingException e) { + // Should never happen, UTF-8 is always supported + throw new RuntimeException(e); + } + } + + // add `language` to the URL query string + if (getLanguage() != null) { + try { + joiner.add( + String.format( + "%slanguage%s=%s", + prefix, + suffix, + URLEncoder.encode(String.valueOf(getLanguage()), "UTF-8") + .replaceAll("\\+", "%20"))); + } catch (UnsupportedEncodingException e) { + // Should never happen, UTF-8 is always supported + throw new RuntimeException(e); + } + } + + // add `max_token_length` to the URL query string + if (getMaxTokenLength() != null) { try { joiner.add( String.format( - "%snum_partitions%s=%s", + "%smax_token_length%s=%s", prefix, suffix, - URLEncoder.encode(String.valueOf(getNumPartitions()), "UTF-8") + URLEncoder.encode(String.valueOf(getMaxTokenLength()), "UTF-8") .replaceAll("\\+", "%20"))); } catch (UnsupportedEncodingException e) { // Should never happen, UTF-8 is always supported @@ -596,15 +737,15 @@ public String toUrlQueryString(String prefix) { } } - // add `num_sub_vectors` to the URL query string - if (getNumSubVectors() != null) { + // add `lower_case` to the URL query string + if (getLowerCase() != null) { try { joiner.add( String.format( - "%snum_sub_vectors%s=%s", + "%slower_case%s=%s", prefix, suffix, - URLEncoder.encode(String.valueOf(getNumSubVectors()), "UTF-8") + URLEncoder.encode(String.valueOf(getLowerCase()), "UTF-8") .replaceAll("\\+", "%20"))); } catch (UnsupportedEncodingException e) { // Should never happen, UTF-8 is always supported @@ -612,30 +753,30 @@ public String toUrlQueryString(String prefix) { } } - // add `num_bits` to the URL query string - if (getNumBits() != null) { + // add `stem` to the URL query string + if (getStem() != null) { try { joiner.add( String.format( - "%snum_bits%s=%s", + "%sstem%s=%s", prefix, suffix, - URLEncoder.encode(String.valueOf(getNumBits()), "UTF-8").replaceAll("\\+", "%20"))); + URLEncoder.encode(String.valueOf(getStem()), "UTF-8").replaceAll("\\+", "%20"))); } catch (UnsupportedEncodingException e) { // Should never happen, UTF-8 is always supported throw new RuntimeException(e); } } - // add `max_iterations` to the URL query string - if (getMaxIterations() != null) { + // add `remove_stop_words` to the URL query string + if (getRemoveStopWords() != null) { try { joiner.add( String.format( - "%smax_iterations%s=%s", + "%sremove_stop_words%s=%s", prefix, suffix, - URLEncoder.encode(String.valueOf(getMaxIterations()), "UTF-8") + URLEncoder.encode(String.valueOf(getRemoveStopWords()), "UTF-8") .replaceAll("\\+", "%20"))); } catch (UnsupportedEncodingException e) { // Should never happen, UTF-8 is always supported @@ -643,15 +784,15 @@ public String toUrlQueryString(String prefix) { } } - // add `sample_rate` to the URL query string - if (getSampleRate() != null) { + // add `ascii_folding` to the URL query string + if (getAsciiFolding() != null) { try { joiner.add( String.format( - "%ssample_rate%s=%s", + "%sascii_folding%s=%s", prefix, suffix, - URLEncoder.encode(String.valueOf(getSampleRate()), "UTF-8") + URLEncoder.encode(String.valueOf(getAsciiFolding()), "UTF-8") .replaceAll("\\+", "%20"))); } catch (UnsupportedEncodingException e) { // Should never happen, UTF-8 is always supported diff --git a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/MatchQuery.java b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/MatchQuery.java index f13998251..c3f4f877e 100644 --- a/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/MatchQuery.java +++ b/java/lance-namespace-apache-client/src/main/java/com/lancedb/lance/namespace/model/MatchQuery.java @@ -43,7 +43,7 @@ public class MatchQuery { @javax.annotation.Nullable private Float boost; public static final String JSON_PROPERTY_COLUMN = "column"; - @javax.annotation.Nullable private JsonNullable column = JsonNullable.undefined(); + @javax.annotation.Nullable private String column; public static final String JSON_PROPERTY_FUZZINESS = "fuzziness"; @@ -89,8 +89,8 @@ public void setBoost(@javax.annotation.Nullable Float boost) { } public MatchQuery column(@javax.annotation.Nullable String column) { - this.column = JsonNullable.of(column); + this.column = column; return this; } @@ -100,24 +100,16 @@ public MatchQuery column(@javax.annotation.Nullable String column) { * @return column */ @javax.annotation.Nullable - @JsonIgnore - public String getColumn() { - return column.orElse(null); - } - @JsonProperty(JSON_PROPERTY_COLUMN) - @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) - public JsonNullable getColumn_JsonNullable() { + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public String getColumn() { return column; } @JsonProperty(JSON_PROPERTY_COLUMN) - public void setColumn_JsonNullable(JsonNullable column) { - this.column = column; - } - + @JsonInclude(value = JsonInclude.Include.ALWAYS) public void setColumn(@javax.annotation.Nullable String column) { - this.column = JsonNullable.of(column); + this.column = column; } public MatchQuery fuzziness(@javax.annotation.Nullable Integer fuzziness) { @@ -260,7 +252,7 @@ public boolean equals(Object o) { } MatchQuery matchQuery = (MatchQuery) o; return Objects.equals(this.boost, matchQuery.boost) - && equalsNullable(this.column, matchQuery.column) + && Objects.equals(this.column, matchQuery.column) && equalsNullable(this.fuzziness, matchQuery.fuzziness) && Objects.equals(this.maxExpansions, matchQuery.maxExpansions) && Objects.equals(this.operator, matchQuery.operator) @@ -280,13 +272,7 @@ private static boolean equalsNullable(JsonNullable a, JsonNullable b) @Override public int hashCode() { return Objects.hash( - boost, - hashCodeNullable(column), - hashCodeNullable(fuzziness), - maxExpansions, - operator, - prefixLength, - terms); + boost, column, hashCodeNullable(fuzziness), maxExpansions, operator, prefixLength, terms); } private static int hashCodeNullable(JsonNullable a) { diff --git a/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespace.java b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespace.java index fffa0f174..9087eb723 100644 --- a/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespace.java +++ b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespace.java @@ -18,6 +18,7 @@ import com.lancedb.lance.namespace.client.apache.api.NamespaceApi; import com.lancedb.lance.namespace.client.apache.api.TableApi; import com.lancedb.lance.namespace.client.apache.api.TransactionApi; +import com.lancedb.lance.namespace.jackson.LanceNamespaceJacksonModule; import com.lancedb.lance.namespace.model.AlterTransactionRequest; import com.lancedb.lance.namespace.model.AlterTransactionResponse; import com.lancedb.lance.namespace.model.CountRowsRequest; @@ -59,6 +60,8 @@ import com.lancedb.lance.namespace.model.UpdateTableRequest; import com.lancedb.lance.namespace.model.UpdateTableResponse; +import com.fasterxml.jackson.databind.ObjectMapper; + import java.util.Map; public class LanceRestNamespace implements LanceNamespace { @@ -69,6 +72,11 @@ public class LanceRestNamespace implements LanceNamespace { private final RestConfig config; public LanceRestNamespace(ApiClient client, Map config) { + // Register custom serializers before creating API instances + ObjectMapper objectMapper = client.getObjectMapper(); + objectMapper.registerModule(new LanceNamespaceJacksonModule()); + client.setObjectMapper(objectMapper); + this.namespaceApi = new NamespaceApi(client); this.tableApi = new TableApi(client); this.transactionApi = new TransactionApi(client); diff --git a/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/jackson/LanceNamespaceJacksonModule.java b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/jackson/LanceNamespaceJacksonModule.java new file mode 100644 index 000000000..07e8caa5e --- /dev/null +++ b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/jackson/LanceNamespaceJacksonModule.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.jackson; + +import com.lancedb.lance.namespace.model.QueryRequest; + +import com.fasterxml.jackson.databind.module.SimpleModule; + +/** Jackson module that registers custom serializers for LanceDB namespace models. */ +public class LanceNamespaceJacksonModule extends SimpleModule { + + public LanceNamespaceJacksonModule() { + super("LanceNamespaceJacksonModule"); + + // Register the custom serializer for QueryRequest + addSerializer(QueryRequest.class, new QueryRequestSerializer()); + } +} diff --git a/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/jackson/QueryRequestSerializer.java b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/jackson/QueryRequestSerializer.java new file mode 100644 index 000000000..7e8149011 --- /dev/null +++ b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/jackson/QueryRequestSerializer.java @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.jackson; + +import com.lancedb.lance.namespace.model.QueryRequest; +import com.lancedb.lance.namespace.model.QueryRequestFullTextQuery; +import com.lancedb.lance.namespace.model.QueryRequestVector; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; +import java.util.List; + +/** + * Custom serializer for QueryRequest that handles the vector field as an untagged union. The server + * expects the vector field as a direct array, not as an object with properties. + */ +public class QueryRequestSerializer extends JsonSerializer { + + @Override + public void serialize(QueryRequest value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + + gen.writeStartObject(); + + // Write required fields + gen.writeStringField("name", value.getName()); + gen.writeObjectField("namespace", value.getNamespace()); + gen.writeNumberField("k", value.getK()); + + // Handle the vector field - server expects empty array, not null + gen.writeFieldName("vector"); + + if (value.getVector() != null) { + QueryRequestVector vectorObj = value.getVector(); + + // Check if it's a single vector or multiple vectors + // Since both fields are initialized as empty lists, we need to check which one has content + if (vectorObj.getMultiVector() != null && !vectorObj.getMultiVector().isEmpty()) { + // Multiple vectors - write as array of arrays + gen.writeStartArray(); + for (List vec : vectorObj.getMultiVector()) { + gen.writeStartArray(); + for (Float elem : vec) { + gen.writeNumber(elem); + } + gen.writeEndArray(); + } + gen.writeEndArray(); + } else if (vectorObj.getSingleVector() != null && !vectorObj.getSingleVector().isEmpty()) { + // Single vector - write as direct array + gen.writeStartArray(); + for (Float elem : vectorObj.getSingleVector()) { + gen.writeNumber(elem); + } + gen.writeEndArray(); + } else { + // If both are empty or null, write empty array as default + gen.writeStartArray(); + gen.writeEndArray(); + } + } else { + // Vector is null, write empty array + gen.writeStartArray(); + gen.writeEndArray(); + } + + // Write optional fields + if (value.getBypassVectorIndex() != null) { + gen.writeBooleanField("bypass_vector_index", value.getBypassVectorIndex()); + } + if (value.getColumns() != null) { + gen.writeObjectField("columns", value.getColumns()); + } + if (value.getDistanceType() != null) { + gen.writeStringField("distance_type", value.getDistanceType()); + } + if (value.getEf() != null) { + gen.writeNumberField("ef", value.getEf()); + } + if (value.getFastSearch() != null) { + gen.writeBooleanField("fast_search", value.getFastSearch()); + } + if (value.getFilter() != null) { + gen.writeStringField("filter", value.getFilter()); + } + if (value.getFullTextQuery() != null) { + gen.writeFieldName("full_text_query"); + + QueryRequestFullTextQuery ftq = value.getFullTextQuery(); + + // Check if it's a string query or structured query and serialize the inner object directly + if (ftq.getStringQuery() != null) { + // Write the StringFtsQuery directly + gen.writeObject(ftq.getStringQuery()); + } else if (ftq.getStructuredQuery() != null) { + // Write the StructuredFtsQuery directly + gen.writeObject(ftq.getStructuredQuery()); + } else { + gen.writeNull(); + } + } + if (value.getLowerBound() != null) { + gen.writeNumberField("lower_bound", value.getLowerBound()); + } + if (value.getNprobes() != null) { + gen.writeNumberField("nprobes", value.getNprobes()); + } + if (value.getOffset() != null) { + gen.writeNumberField("offset", value.getOffset()); + } + if (value.getPrefilter() != null) { + gen.writeBooleanField("prefilter", value.getPrefilter()); + } + if (value.getRefineFactor() != null) { + gen.writeNumberField("refine_factor", value.getRefineFactor()); + } + if (value.getUpperBound() != null) { + gen.writeNumberField("upper_bound", value.getUpperBound()); + } + if (value.getVectorColumn() != null) { + gen.writeStringField("vector_column", value.getVectorColumn()); + } + if (value.getVersion() != null) { + gen.writeNumberField("version", value.getVersion()); + } + if (value.getWithRowId() != null) { + gen.writeBooleanField("with_row_id", value.getWithRowId()); + } + + gen.writeEndObject(); + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java index 53ae358d9..7cc2faf75 100644 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/index/IndexTest.java @@ -22,7 +22,9 @@ import java.io.IOException; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** Tests for index operations: create, list, stats. */ public class IndexTest extends BaseNamespaceTest { @@ -217,7 +219,7 @@ public void testMultipleIndices() throws IOException, InterruptedException { System.out.println("\n--- Creating FTS index ---"); CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); ftsIndexRequest.setName(tableName); - ftsIndexRequest.setColumn("text"); + ftsIndexRequest.setColumn("name"); ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); namespace.createIndex(ftsIndexRequest); @@ -245,7 +247,7 @@ public void testMultipleIndices() throws IOException, InterruptedException { boolean hasScalarIndex = listResponse.getIndexes().stream().anyMatch(idx -> idx.getIndexName().equals("id_idx")); boolean hasFtsIndex = - listResponse.getIndexes().stream().anyMatch(idx -> idx.getIndexName().equals("text_idx")); + listResponse.getIndexes().stream().anyMatch(idx -> idx.getIndexName().equals("name_idx")); assertTrue(hasVectorIndex, "Should have vector index"); assertTrue(hasScalarIndex, "Should have scalar index"); diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/jackson/DebugSerializerTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/jackson/DebugSerializerTest.java new file mode 100644 index 000000000..cc5bf2d86 --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/jackson/DebugSerializerTest.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.jackson; + +import com.lancedb.lance.namespace.client.apache.ApiClient; +import com.lancedb.lance.namespace.jackson.LanceNamespaceJacksonModule; +import com.lancedb.lance.namespace.model.QueryRequest; +import com.lancedb.lance.namespace.model.QueryRequestFullTextQuery; +import com.lancedb.lance.namespace.model.StringFtsQuery; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +public class DebugSerializerTest { + + @Test + public void debugFullTextQuerySerialization() throws Exception { + // Create the same structure as in the failing test + QueryRequest ftsPrefilterQuery = new QueryRequest(); + ftsPrefilterQuery.setName("test_table"); + ftsPrefilterQuery.setNamespace(Arrays.asList("test")); + ftsPrefilterQuery.setK(20); + ftsPrefilterQuery.setPrefilter(true); + ftsPrefilterQuery.setFilter("id < 25"); + ftsPrefilterQuery.setColumns(Arrays.asList("id", "text")); + + StringFtsQuery fts = new StringFtsQuery(); + fts.setQuery("document"); + QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); + fullTextQuery.setStringQuery(fts); + ftsPrefilterQuery.setFullTextQuery(fullTextQuery); + + // Test with default ObjectMapper (no custom serializer) + ObjectMapper defaultMapper = new ObjectMapper(); + String defaultJson = defaultMapper.writeValueAsString(ftsPrefilterQuery); + System.out.println("Default JSON (without custom serializer):"); + System.out.println(defaultJson); + System.out.println(); + + // Test with custom serializer + ObjectMapper customMapper = new ObjectMapper(); + customMapper.registerModule(new LanceNamespaceJacksonModule()); + String customJson = customMapper.writeValueAsString(ftsPrefilterQuery); + System.out.println("Custom JSON (with custom serializer):"); + System.out.println(customJson); + System.out.println(); + + // Test with ApiClient's ObjectMapper + ApiClient apiClient = new ApiClient(); + ObjectMapper apiClientMapper = apiClient.getObjectMapper(); + apiClientMapper.registerModule(new LanceNamespaceJacksonModule()); + String apiClientJson = apiClientMapper.writeValueAsString(ftsPrefilterQuery); + System.out.println("ApiClient JSON (with custom serializer):"); + System.out.println(apiClientJson); + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/jackson/QueryRequestSerializerTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/jackson/QueryRequestSerializerTest.java new file mode 100644 index 000000000..adc4e83b3 --- /dev/null +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/jackson/QueryRequestSerializerTest.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.test.jackson; + +import com.lancedb.lance.namespace.jackson.LanceNamespaceJacksonModule; +import com.lancedb.lance.namespace.model.QueryRequest; +import com.lancedb.lance.namespace.model.QueryRequestVector; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class QueryRequestSerializerTest { + + private ObjectMapper objectMapper; + + @BeforeEach + public void setUp() { + objectMapper = new ObjectMapper(); + objectMapper.registerModule(new LanceNamespaceJacksonModule()); + } + + @Test + public void testSingleVectorSerialization() throws Exception { + // Create a query request with a single vector + QueryRequest request = new QueryRequest(); + request.setName("test_table"); + request.setNamespace(Arrays.asList("test", "namespace")); + request.setK(10); + + // Set single vector + List vector = Arrays.asList(1.0f, 2.0f, 3.0f, 4.0f); + QueryRequestVector vectorObj = new QueryRequestVector(); + vectorObj.setSingleVector(vector); + request.setVector(vectorObj); + + // Serialize to JSON + String json = objectMapper.writeValueAsString(request); + System.out.println("Serialized JSON: " + json); + + // Verify the vector is serialized as a direct array + assertTrue(json.contains("\"vector\":[1.0,2.0,3.0,4.0]")); + assertTrue(json.contains("\"name\":\"test_table\"")); + assertTrue(json.contains("\"k\":10")); + } + + @Test + public void testMultiVectorSerialization() throws Exception { + // Create a query request with multiple vectors + QueryRequest request = new QueryRequest(); + request.setName("test_table"); + request.setNamespace(Arrays.asList("test", "namespace")); + request.setK(5); + + // Set multiple vectors + List> multiVector = + Arrays.asList( + Arrays.asList(1.0f, 2.0f), Arrays.asList(3.0f, 4.0f), Arrays.asList(5.0f, 6.0f)); + QueryRequestVector vectorObj = new QueryRequestVector(); + vectorObj.setMultiVector(multiVector); + request.setVector(vectorObj); + + // Serialize to JSON + String json = objectMapper.writeValueAsString(request); + System.out.println("Serialized JSON: " + json); + + // Verify the vector is serialized as an array of arrays + assertTrue(json.contains("\"vector\":[[1.0,2.0],[3.0,4.0],[5.0,6.0]]")); + } + + @Test + public void testOptionalFieldsSerialization() throws Exception { + // Create a query request with optional fields + QueryRequest request = new QueryRequest(); + request.setName("test_table"); + request.setNamespace(Arrays.asList("test")); + request.setK(10); + + // Set vector + QueryRequestVector vectorObj = new QueryRequestVector(); + vectorObj.setSingleVector(Arrays.asList(1.0f)); + request.setVector(vectorObj); + + // Set optional fields + request.setFilter("id > 10"); + request.setPrefilter(true); + request.setColumns(Arrays.asList("id", "name")); + + // Serialize to JSON + String json = objectMapper.writeValueAsString(request); + System.out.println("Serialized JSON: " + json); + + // Verify optional fields are included + assertTrue(json.contains("\"filter\":\"id > 10\"")); + assertTrue(json.contains("\"prefilter\":true")); + assertTrue(json.contains("\"columns\":[\"id\",\"name\"]")); + } + + @Test + public void testEmptyVectorSerialization() throws Exception { + // Create a query request with an empty vector + QueryRequest request = new QueryRequest(); + request.setName("test_table"); + request.setNamespace(Arrays.asList("test")); + request.setK(10); + + // Set empty vector + QueryRequestVector vectorObj = new QueryRequestVector(); + vectorObj.setSingleVector(new ArrayList<>()); // Empty list of Float + request.setVector(vectorObj); + + // Serialize to JSON + ObjectMapper customMapper = new ObjectMapper(); + customMapper.registerModule(new LanceNamespaceJacksonModule()); + String json = customMapper.writeValueAsString(request); + System.out.println("Serialized JSON with empty vector: " + json); + + // Verify the vector is serialized as an empty array + assertTrue(json.contains("\"vector\":[]")); + } + + @Test + public void testNullVectorSerialization() throws Exception { + // Create a query request with null vector + QueryRequest request = new QueryRequest(); + request.setName("test_table"); + request.setNamespace(Arrays.asList("test")); + request.setK(10); + // Don't set vector at all (leave it null) + + // Serialize to JSON + ObjectMapper customMapper = new ObjectMapper(); + customMapper.registerModule(new LanceNamespaceJacksonModule()); + String json = customMapper.writeValueAsString(request); + System.out.println("Serialized JSON with null vector: " + json); + + // Verify the vector is serialized as an empty array + assertTrue(json.contains("\"vector\":[]")); + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java index ccf02be9d..2faa2f4e8 100644 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/query/FullTextSearchTest.java @@ -25,7 +25,9 @@ import java.util.Arrays; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** Tests for full-text search functionality. */ public class FullTextSearchTest extends BaseNamespaceTest { @@ -178,6 +180,11 @@ public void testFullTextSearchWithFilter() throws IOException, InterruptedExcept fullTextQuery.setStringQuery(fts); ftsPrefilterQuery.setFullTextQuery(fullTextQuery); + // Add empty vector to satisfy API requirement + QueryRequestVector emptyVector = new QueryRequestVector(); + emptyVector.setSingleVector(new ArrayList<>()); + ftsPrefilterQuery.setVector(emptyVector); + byte[] ftsPrefilterResult = namespace.queryTable(ftsPrefilterQuery); assertNotNull(ftsPrefilterResult, "FTS prefilter query result should not be null"); @@ -254,11 +261,12 @@ public void testStructuredFullTextSearch() throws IOException, InterruptedExcept namespace.createTable(tableName, tableData); - // Create FTS index + // Create FTS index with position for phrase queries CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); ftsIndexRequest.setName(tableName); ftsIndexRequest.setColumn("text"); ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); + ftsIndexRequest.setWithPosition(true); // Required for phrase queries namespace.createIndex(ftsIndexRequest); TestUtils.waitForIndexComplete(namespace, tableName, "text_idx", 30); @@ -288,6 +296,7 @@ public void testStructuredFullTextSearch() throws IOException, InterruptedExcept FtsQuery shouldQuery = new FtsQuery(); MatchQuery shouldMatch = new MatchQuery(); shouldMatch.setTerms("document"); + shouldMatch.setColumn("text"); shouldQuery.setMatch(shouldMatch); boolQuery.setShould(Arrays.asList(shouldQuery)); @@ -334,119 +343,15 @@ public void testStructuredFullTextSearch() throws IOException, InterruptedExcept int phraseRowCount = ArrowTestUtils.countRows(phraseResult, allocator); assertEquals( - 5, phraseRowCount, "Should find exactly 5 rows with 'important' near 'document'"); + 4, phraseRowCount, "Should find exactly 4 rows with 'important' near 'document'"); List phraseIds = ArrowTestUtils.extractColumn(phraseResult, allocator, "id", Integer.class); - for (int i = 1; i <= 5; i++) { - assertTrue(phraseIds.contains(i), "Should find row " + i); - } - } finally { - TestUtils.dropTable(namespace, tableName); - } - } - - @Test - public void testHybridSearch() throws IOException, InterruptedException { - skipIfNotConfigured(); - - System.out.println("=== Test: Hybrid Search (Vector + Full-Text) ==="); - String tableName = TestUtils.generateTableName("test_hybrid"); - - try { - // Create table with text and vector data - System.out.println("\n--- Creating table with text and vector data ---"); - ArrowTestUtils.TableDataBuilder builder = - new ArrowTestUtils.TableDataBuilder(allocator) - .withSchema(ArrowTestUtils.createSchemaWithText(128)); - - // Add 20 rows with specific patterns - for (int i = 1; i <= 20; i++) { - if (i <= 5) { - builder.withText(i, "Important document about vector search"); - } else if (i <= 10) { - builder.withText(i, "Regular document with some content"); - } else if (i <= 15) { - builder.withText(i, "Basic text without keywords"); - } else { - builder.withText(i, "Another document for testing"); - } - } - - byte[] tableData = builder.addRows(1, 20).build(); - namespace.createTable(tableName, tableData); - - // Create FTS index - CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); - ftsIndexRequest.setName(tableName); - ftsIndexRequest.setColumn("text"); - ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); - namespace.createIndex(ftsIndexRequest); - TestUtils.waitForIndexComplete(namespace, tableName, "text_idx", 30); - - // Hybrid search: vector + text - System.out.println("\n--- Testing hybrid search ---"); - QueryRequest hybridQuery = new QueryRequest(); - hybridQuery.setName(tableName); - hybridQuery.setColumns(Arrays.asList("id", "text")); - - // Add vector search - search for vector with all 10s - List vectorList = new ArrayList<>(); - for (int i = 0; i < 128; i++) { - vectorList.add(10.0f); - } - - // Wrap the vector in QueryRequestVector - QueryRequestVector queryVector = new QueryRequestVector(); - queryVector.setSingleVector(vectorList); - hybridQuery.setVector(queryVector); - hybridQuery.setK(10); - - // Add full-text search - StringFtsQuery ftsQuery = new StringFtsQuery(); - ftsQuery.setQuery("document"); - QueryRequestFullTextQuery fullTextQuery = new QueryRequestFullTextQuery(); - fullTextQuery.setStringQuery(ftsQuery); - hybridQuery.setFullTextQuery(fullTextQuery); - - byte[] hybridResult = namespace.queryTable(hybridQuery); - assertNotNull(hybridResult, "Hybrid search result should not be null"); - - int hybridRowCount = ArrowTestUtils.countRows(hybridResult, allocator); - assertEquals( - 10, hybridRowCount, "Should find exactly 10 rows (1-10, 16-20 contain 'document')"); - - List hybridIds = - ArrowTestUtils.extractColumn(hybridResult, allocator, "id", Integer.class); - - // Verify all document-containing rows are found - for (int i = 1; i <= 10; i++) { - assertTrue(hybridIds.contains(i), "Should find row " + i); - } - for (int i = 16; i <= 20; i++) { - assertTrue(hybridIds.contains(i), "Should find row " + i); - } - // Hybrid search with filter - System.out.println("\n--- Testing hybrid search with filter (id <= 10) ---"); - QueryRequest hybridFilterQuery = new QueryRequest(); - hybridFilterQuery.setName(tableName); - hybridFilterQuery.setVector(queryVector); // Reuse the same QueryRequestVector from above - hybridFilterQuery.setK(20); - hybridFilterQuery.setFilter("id <= 10"); - hybridFilterQuery.setFullTextQuery(fullTextQuery); - hybridFilterQuery.setColumns(Arrays.asList("id", "text")); - - byte[] hybridFilterResult = namespace.queryTable(hybridFilterQuery); - assertNotNull(hybridFilterResult, "Hybrid search with filter result should not be null"); - - List ids = - ArrowTestUtils.extractColumn(hybridFilterResult, allocator, "id", Integer.class); - assertEquals(10, ids.size(), "Should find exactly 10 rows (filter limits to id <= 10)"); - assertTrue(ids.stream().allMatch(id -> id <= 10), "All IDs should be <= 10"); - - for (int i = 1; i <= 10; i++) { - assertTrue(ids.contains(i), "Should find row " + i); - } + // Rows 1, 2, 3, 5 should match (row 4 has terms in wrong order) + assertTrue(phraseIds.contains(1), "Should find row 1"); + assertTrue(phraseIds.contains(2), "Should find row 2"); + assertTrue(phraseIds.contains(3), "Should find row 3"); + assertTrue(phraseIds.contains(5), "Should find row 5"); } finally { TestUtils.dropTable(namespace, tableName); } diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java index 66a959887..1438ff909 100644 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/utils/TestUtils.java @@ -134,8 +134,6 @@ public static QueryRequest createVectorQuery( for (int i = 0; i < dimensions; i++) { vector.add(targetValue); } - - // Wrap the vector in QueryRequestVector QueryRequestVector queryVector = new QueryRequestVector(); queryVector.setSingleVector(vector); query.setVector(queryVector); diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/CreateIndexRequest.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/CreateIndexRequest.java index 77a3844b4..07e6031b3 100644 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/CreateIndexRequest.java +++ b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/CreateIndexRequest.java @@ -121,15 +121,21 @@ public static MetricTypeEnum fromValue(String value) { private MetricTypeEnum metricType; - private Integer numPartitions; + private Boolean withPosition; - private Integer numSubVectors; + private String baseTokenizer; - private Integer numBits; + private String language; - private Integer maxIterations; + private Integer maxTokenLength; - private Integer sampleRate; + private Boolean lowerCase; + + private Boolean stem; + + private Boolean removeStopWords; + + private Boolean asciiFolding; public CreateIndexRequest() { super(); @@ -271,125 +277,189 @@ public void setMetricType(MetricTypeEnum metricType) { this.metricType = metricType; } - public CreateIndexRequest numPartitions(Integer numPartitions) { - this.numPartitions = numPartitions; + public CreateIndexRequest withPosition(Boolean withPosition) { + this.withPosition = withPosition; + return this; + } + + /** + * Optional FTS parameter for position tracking + * + * @return withPosition + */ + @Schema( + name = "with_position", + description = "Optional FTS parameter for position tracking", + requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("with_position") + public Boolean getWithPosition() { + return withPosition; + } + + public void setWithPosition(Boolean withPosition) { + this.withPosition = withPosition; + } + + public CreateIndexRequest baseTokenizer(String baseTokenizer) { + this.baseTokenizer = baseTokenizer; + return this; + } + + /** + * Optional FTS parameter for base tokenizer + * + * @return baseTokenizer + */ + @Schema( + name = "base_tokenizer", + description = "Optional FTS parameter for base tokenizer", + requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("base_tokenizer") + public String getBaseTokenizer() { + return baseTokenizer; + } + + public void setBaseTokenizer(String baseTokenizer) { + this.baseTokenizer = baseTokenizer; + } + + public CreateIndexRequest language(String language) { + this.language = language; + return this; + } + + /** + * Optional FTS parameter for language + * + * @return language + */ + @Schema( + name = "language", + description = "Optional FTS parameter for language", + requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @JsonProperty("language") + public String getLanguage() { + return language; + } + + public void setLanguage(String language) { + this.language = language; + } + + public CreateIndexRequest maxTokenLength(Integer maxTokenLength) { + this.maxTokenLength = maxTokenLength; return this; } /** - * Number of partitions for IVF indexes minimum: 1 + * Optional FTS parameter for maximum token length minimum: 0 * - * @return numPartitions + * @return maxTokenLength */ - @Min(1) + @Min(0) @Schema( - name = "num_partitions", - description = "Number of partitions for IVF indexes", + name = "max_token_length", + description = "Optional FTS parameter for maximum token length", requiredMode = Schema.RequiredMode.NOT_REQUIRED) - @JsonProperty("num_partitions") - public Integer getNumPartitions() { - return numPartitions; + @JsonProperty("max_token_length") + public Integer getMaxTokenLength() { + return maxTokenLength; } - public void setNumPartitions(Integer numPartitions) { - this.numPartitions = numPartitions; + public void setMaxTokenLength(Integer maxTokenLength) { + this.maxTokenLength = maxTokenLength; } - public CreateIndexRequest numSubVectors(Integer numSubVectors) { - this.numSubVectors = numSubVectors; + public CreateIndexRequest lowerCase(Boolean lowerCase) { + this.lowerCase = lowerCase; return this; } /** - * Number of sub-vectors for PQ indexes minimum: 1 + * Optional FTS parameter for lowercase conversion * - * @return numSubVectors + * @return lowerCase */ - @Min(1) @Schema( - name = "num_sub_vectors", - description = "Number of sub-vectors for PQ indexes", + name = "lower_case", + description = "Optional FTS parameter for lowercase conversion", requiredMode = Schema.RequiredMode.NOT_REQUIRED) - @JsonProperty("num_sub_vectors") - public Integer getNumSubVectors() { - return numSubVectors; + @JsonProperty("lower_case") + public Boolean getLowerCase() { + return lowerCase; } - public void setNumSubVectors(Integer numSubVectors) { - this.numSubVectors = numSubVectors; + public void setLowerCase(Boolean lowerCase) { + this.lowerCase = lowerCase; } - public CreateIndexRequest numBits(Integer numBits) { - this.numBits = numBits; + public CreateIndexRequest stem(Boolean stem) { + this.stem = stem; return this; } /** - * Number of bits for scalar quantization minimum: 1 maximum: 8 + * Optional FTS parameter for stemming * - * @return numBits + * @return stem */ - @Min(1) - @Max(8) @Schema( - name = "num_bits", - description = "Number of bits for scalar quantization", + name = "stem", + description = "Optional FTS parameter for stemming", requiredMode = Schema.RequiredMode.NOT_REQUIRED) - @JsonProperty("num_bits") - public Integer getNumBits() { - return numBits; + @JsonProperty("stem") + public Boolean getStem() { + return stem; } - public void setNumBits(Integer numBits) { - this.numBits = numBits; + public void setStem(Boolean stem) { + this.stem = stem; } - public CreateIndexRequest maxIterations(Integer maxIterations) { - this.maxIterations = maxIterations; + public CreateIndexRequest removeStopWords(Boolean removeStopWords) { + this.removeStopWords = removeStopWords; return this; } /** - * Maximum iterations for index building minimum: 1 + * Optional FTS parameter for stop word removal * - * @return maxIterations + * @return removeStopWords */ - @Min(1) @Schema( - name = "max_iterations", - description = "Maximum iterations for index building", + name = "remove_stop_words", + description = "Optional FTS parameter for stop word removal", requiredMode = Schema.RequiredMode.NOT_REQUIRED) - @JsonProperty("max_iterations") - public Integer getMaxIterations() { - return maxIterations; + @JsonProperty("remove_stop_words") + public Boolean getRemoveStopWords() { + return removeStopWords; } - public void setMaxIterations(Integer maxIterations) { - this.maxIterations = maxIterations; + public void setRemoveStopWords(Boolean removeStopWords) { + this.removeStopWords = removeStopWords; } - public CreateIndexRequest sampleRate(Integer sampleRate) { - this.sampleRate = sampleRate; + public CreateIndexRequest asciiFolding(Boolean asciiFolding) { + this.asciiFolding = asciiFolding; return this; } /** - * Sample rate for index building minimum: 1 + * Optional FTS parameter for ASCII folding * - * @return sampleRate + * @return asciiFolding */ - @Min(1) @Schema( - name = "sample_rate", - description = "Sample rate for index building", + name = "ascii_folding", + description = "Optional FTS parameter for ASCII folding", requiredMode = Schema.RequiredMode.NOT_REQUIRED) - @JsonProperty("sample_rate") - public Integer getSampleRate() { - return sampleRate; + @JsonProperty("ascii_folding") + public Boolean getAsciiFolding() { + return asciiFolding; } - public void setSampleRate(Integer sampleRate) { - this.sampleRate = sampleRate; + public void setAsciiFolding(Boolean asciiFolding) { + this.asciiFolding = asciiFolding; } @Override @@ -406,11 +476,14 @@ public boolean equals(Object o) { && Objects.equals(this.column, createIndexRequest.column) && Objects.equals(this.indexType, createIndexRequest.indexType) && Objects.equals(this.metricType, createIndexRequest.metricType) - && Objects.equals(this.numPartitions, createIndexRequest.numPartitions) - && Objects.equals(this.numSubVectors, createIndexRequest.numSubVectors) - && Objects.equals(this.numBits, createIndexRequest.numBits) - && Objects.equals(this.maxIterations, createIndexRequest.maxIterations) - && Objects.equals(this.sampleRate, createIndexRequest.sampleRate); + && Objects.equals(this.withPosition, createIndexRequest.withPosition) + && Objects.equals(this.baseTokenizer, createIndexRequest.baseTokenizer) + && Objects.equals(this.language, createIndexRequest.language) + && Objects.equals(this.maxTokenLength, createIndexRequest.maxTokenLength) + && Objects.equals(this.lowerCase, createIndexRequest.lowerCase) + && Objects.equals(this.stem, createIndexRequest.stem) + && Objects.equals(this.removeStopWords, createIndexRequest.removeStopWords) + && Objects.equals(this.asciiFolding, createIndexRequest.asciiFolding); } @Override @@ -421,11 +494,14 @@ public int hashCode() { column, indexType, metricType, - numPartitions, - numSubVectors, - numBits, - maxIterations, - sampleRate); + withPosition, + baseTokenizer, + language, + maxTokenLength, + lowerCase, + stem, + removeStopWords, + asciiFolding); } @Override @@ -437,11 +513,14 @@ public String toString() { sb.append(" column: ").append(toIndentedString(column)).append("\n"); sb.append(" indexType: ").append(toIndentedString(indexType)).append("\n"); sb.append(" metricType: ").append(toIndentedString(metricType)).append("\n"); - sb.append(" numPartitions: ").append(toIndentedString(numPartitions)).append("\n"); - sb.append(" numSubVectors: ").append(toIndentedString(numSubVectors)).append("\n"); - sb.append(" numBits: ").append(toIndentedString(numBits)).append("\n"); - sb.append(" maxIterations: ").append(toIndentedString(maxIterations)).append("\n"); - sb.append(" sampleRate: ").append(toIndentedString(sampleRate)).append("\n"); + sb.append(" withPosition: ").append(toIndentedString(withPosition)).append("\n"); + sb.append(" baseTokenizer: ").append(toIndentedString(baseTokenizer)).append("\n"); + sb.append(" language: ").append(toIndentedString(language)).append("\n"); + sb.append(" maxTokenLength: ").append(toIndentedString(maxTokenLength)).append("\n"); + sb.append(" lowerCase: ").append(toIndentedString(lowerCase)).append("\n"); + sb.append(" stem: ").append(toIndentedString(stem)).append("\n"); + sb.append(" removeStopWords: ").append(toIndentedString(removeStopWords)).append("\n"); + sb.append(" asciiFolding: ").append(toIndentedString(asciiFolding)).append("\n"); sb.append("}"); return sb.toString(); } diff --git a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/MatchQuery.java b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/MatchQuery.java index 4d981c9a1..5caad02ce 100644 --- a/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/MatchQuery.java +++ b/java/lance-namespace-springboot-server/src/main/java/com/lancedb/lance/namespace/server/springboot/model/MatchQuery.java @@ -48,7 +48,8 @@ public MatchQuery() { } /** Constructor with only required parameters */ - public MatchQuery(String terms) { + public MatchQuery(String column, String terms) { + this.column = column; this.terms = terms; } @@ -82,7 +83,8 @@ public MatchQuery column(String column) { * * @return column */ - @Schema(name = "column", requiredMode = Schema.RequiredMode.NOT_REQUIRED) + @NotNull + @Schema(name = "column", requiredMode = Schema.RequiredMode.REQUIRED) @JsonProperty("column") public String getColumn() { return column; diff --git a/python/lance_namespace_urllib3_client/docs/CreateIndexRequest.md b/python/lance_namespace_urllib3_client/docs/CreateIndexRequest.md index 7f456bc30..9fb3bf423 100644 --- a/python/lance_namespace_urllib3_client/docs/CreateIndexRequest.md +++ b/python/lance_namespace_urllib3_client/docs/CreateIndexRequest.md @@ -10,11 +10,14 @@ Name | Type | Description | Notes **column** | **str** | Name of the column to create index on | **index_type** | **str** | Type of index to create | **metric_type** | **str** | Distance metric type for vector indexes | [optional] -**num_partitions** | **int** | Number of partitions for IVF indexes | [optional] -**num_sub_vectors** | **int** | Number of sub-vectors for PQ indexes | [optional] -**num_bits** | **int** | Number of bits for scalar quantization | [optional] -**max_iterations** | **int** | Maximum iterations for index building | [optional] -**sample_rate** | **int** | Sample rate for index building | [optional] +**with_position** | **bool** | Optional FTS parameter for position tracking | [optional] +**base_tokenizer** | **str** | Optional FTS parameter for base tokenizer | [optional] +**language** | **str** | Optional FTS parameter for language | [optional] +**max_token_length** | **int** | Optional FTS parameter for maximum token length | [optional] +**lower_case** | **bool** | Optional FTS parameter for lowercase conversion | [optional] +**stem** | **bool** | Optional FTS parameter for stemming | [optional] +**remove_stop_words** | **bool** | Optional FTS parameter for stop word removal | [optional] +**ascii_folding** | **bool** | Optional FTS parameter for ASCII folding | [optional] ## Example diff --git a/python/lance_namespace_urllib3_client/docs/MatchQuery.md b/python/lance_namespace_urllib3_client/docs/MatchQuery.md index 5da7c9d91..c1b65c61b 100644 --- a/python/lance_namespace_urllib3_client/docs/MatchQuery.md +++ b/python/lance_namespace_urllib3_client/docs/MatchQuery.md @@ -6,7 +6,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **boost** | **float** | | [optional] -**column** | **str** | | [optional] +**column** | **str** | | **fuzziness** | **int** | | [optional] **max_expansions** | **int** | The maximum number of terms to expand for fuzzy matching. Default to 50. | [optional] **operator** | [**Operator**](Operator.md) | The operator to use for combining terms. This can be either `And` or `Or`, it's 'Or' by default. - `And`: All terms must match. - `Or`: At least one term must match. | [optional] diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/create_index_request.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/create_index_request.py index 35d6f9f46..2bc3bc573 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/create_index_request.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/create_index_request.py @@ -17,7 +17,7 @@ import re # noqa: F401 import json -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr, field_validator from typing import Any, ClassVar, Dict, List, Optional from typing_extensions import Annotated from typing import Optional, Set @@ -32,12 +32,15 @@ class CreateIndexRequest(BaseModel): column: StrictStr = Field(description="Name of the column to create index on") index_type: StrictStr = Field(description="Type of index to create") metric_type: Optional[StrictStr] = Field(default=None, description="Distance metric type for vector indexes") - num_partitions: Optional[Annotated[int, Field(strict=True, ge=1)]] = Field(default=None, description="Number of partitions for IVF indexes") - num_sub_vectors: Optional[Annotated[int, Field(strict=True, ge=1)]] = Field(default=None, description="Number of sub-vectors for PQ indexes") - num_bits: Optional[Annotated[int, Field(le=8, strict=True, ge=1)]] = Field(default=None, description="Number of bits for scalar quantization") - max_iterations: Optional[Annotated[int, Field(strict=True, ge=1)]] = Field(default=None, description="Maximum iterations for index building") - sample_rate: Optional[Annotated[int, Field(strict=True, ge=1)]] = Field(default=None, description="Sample rate for index building") - __properties: ClassVar[List[str]] = ["name", "namespace", "column", "index_type", "metric_type", "num_partitions", "num_sub_vectors", "num_bits", "max_iterations", "sample_rate"] + with_position: Optional[StrictBool] = Field(default=None, description="Optional FTS parameter for position tracking") + base_tokenizer: Optional[StrictStr] = Field(default=None, description="Optional FTS parameter for base tokenizer") + language: Optional[StrictStr] = Field(default=None, description="Optional FTS parameter for language") + max_token_length: Optional[Annotated[int, Field(strict=True, ge=0)]] = Field(default=None, description="Optional FTS parameter for maximum token length") + lower_case: Optional[StrictBool] = Field(default=None, description="Optional FTS parameter for lowercase conversion") + stem: Optional[StrictBool] = Field(default=None, description="Optional FTS parameter for stemming") + remove_stop_words: Optional[StrictBool] = Field(default=None, description="Optional FTS parameter for stop word removal") + ascii_folding: Optional[StrictBool] = Field(default=None, description="Optional FTS parameter for ASCII folding") + __properties: ClassVar[List[str]] = ["name", "namespace", "column", "index_type", "metric_type", "with_position", "base_tokenizer", "language", "max_token_length", "lower_case", "stem", "remove_stop_words", "ascii_folding"] @field_validator('index_type') def index_type_validate_enum(cls, value): @@ -112,11 +115,14 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "column": obj.get("column"), "index_type": obj.get("index_type"), "metric_type": obj.get("metric_type"), - "num_partitions": obj.get("num_partitions"), - "num_sub_vectors": obj.get("num_sub_vectors"), - "num_bits": obj.get("num_bits"), - "max_iterations": obj.get("max_iterations"), - "sample_rate": obj.get("sample_rate") + "with_position": obj.get("with_position"), + "base_tokenizer": obj.get("base_tokenizer"), + "language": obj.get("language"), + "max_token_length": obj.get("max_token_length"), + "lower_case": obj.get("lower_case"), + "stem": obj.get("stem"), + "remove_stop_words": obj.get("remove_stop_words"), + "ascii_folding": obj.get("ascii_folding") }) return _obj diff --git a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/match_query.py b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/match_query.py index eb3bec437..c2332d28b 100644 --- a/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/match_query.py +++ b/python/lance_namespace_urllib3_client/lance_namespace_urllib3_client/models/match_query.py @@ -29,7 +29,7 @@ class MatchQuery(BaseModel): MatchQuery """ # noqa: E501 boost: Optional[Union[StrictFloat, StrictInt]] = None - column: Optional[StrictStr] = None + column: Optional[StrictStr] fuzziness: Optional[Annotated[int, Field(strict=True, ge=0)]] = None max_expansions: Optional[Annotated[int, Field(strict=True, ge=0)]] = Field(default=None, description="The maximum number of terms to expand for fuzzy matching. Default to 50.") operator: Optional[Operator] = Field(default=None, description="The operator to use for combining terms. This can be either `And` or `Or`, it's 'Or' by default. - `And`: All terms must match. - `Or`: At least one term must match.") diff --git a/python/lance_namespace_urllib3_client/test/test_create_index_request.py b/python/lance_namespace_urllib3_client/test/test_create_index_request.py index 9a8989e10..bd8b60a70 100644 --- a/python/lance_namespace_urllib3_client/test/test_create_index_request.py +++ b/python/lance_namespace_urllib3_client/test/test_create_index_request.py @@ -42,11 +42,14 @@ def make_instance(self, include_optional) -> CreateIndexRequest: column = '', index_type = 'BTREE', metric_type = 'l2', - num_partitions = 1, - num_sub_vectors = 1, - num_bits = 1, - max_iterations = 1, - sample_rate = 1 + with_position = True, + base_tokenizer = '', + language = '', + max_token_length = 0, + lower_case = True, + stem = True, + remove_stop_words = True, + ascii_folding = True ) else: return CreateIndexRequest( diff --git a/python/lance_namespace_urllib3_client/test/test_match_query.py b/python/lance_namespace_urllib3_client/test/test_match_query.py index 4e69af1ea..f8b33efd3 100644 --- a/python/lance_namespace_urllib3_client/test/test_match_query.py +++ b/python/lance_namespace_urllib3_client/test/test_match_query.py @@ -45,6 +45,7 @@ def make_instance(self, include_optional) -> MatchQuery: ) else: return MatchQuery( + column = '', terms = '', ) """ diff --git a/rust/lance-namespace-reqwest-client/docs/CreateIndexRequest.md b/rust/lance-namespace-reqwest-client/docs/CreateIndexRequest.md index 13a950f2f..9d13e381e 100644 --- a/rust/lance-namespace-reqwest-client/docs/CreateIndexRequest.md +++ b/rust/lance-namespace-reqwest-client/docs/CreateIndexRequest.md @@ -9,11 +9,14 @@ Name | Type | Description | Notes **column** | **String** | Name of the column to create index on | **index_type** | **String** | Type of index to create | **metric_type** | Option<**String**> | Distance metric type for vector indexes | [optional] -**num_partitions** | Option<**i32**> | Number of partitions for IVF indexes | [optional] -**num_sub_vectors** | Option<**i32**> | Number of sub-vectors for PQ indexes | [optional] -**num_bits** | Option<**i32**> | Number of bits for scalar quantization | [optional] -**max_iterations** | Option<**i32**> | Maximum iterations for index building | [optional] -**sample_rate** | Option<**i32**> | Sample rate for index building | [optional] +**with_position** | Option<**bool**> | Optional FTS parameter for position tracking | [optional] +**base_tokenizer** | Option<**String**> | Optional FTS parameter for base tokenizer | [optional] +**language** | Option<**String**> | Optional FTS parameter for language | [optional] +**max_token_length** | Option<**i32**> | Optional FTS parameter for maximum token length | [optional] +**lower_case** | Option<**bool**> | Optional FTS parameter for lowercase conversion | [optional] +**stem** | Option<**bool**> | Optional FTS parameter for stemming | [optional] +**remove_stop_words** | Option<**bool**> | Optional FTS parameter for stop word removal | [optional] +**ascii_folding** | Option<**bool**> | Optional FTS parameter for ASCII folding | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/rust/lance-namespace-reqwest-client/docs/MatchQuery.md b/rust/lance-namespace-reqwest-client/docs/MatchQuery.md index 096d0931b..21452b1c1 100644 --- a/rust/lance-namespace-reqwest-client/docs/MatchQuery.md +++ b/rust/lance-namespace-reqwest-client/docs/MatchQuery.md @@ -5,7 +5,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **boost** | Option<**f32**> | | [optional] -**column** | Option<**String**> | | [optional] +**column** | Option<**String**> | | **fuzziness** | Option<**i32**> | | [optional] **max_expansions** | Option<**i32**> | The maximum number of terms to expand for fuzzy matching. Default to 50. | [optional] **operator** | Option<[**models::Operator**](Operator.md)> | The operator to use for combining terms. This can be either `And` or `Or`, it's 'Or' by default. - `And`: All terms must match. - `Or`: At least one term must match. | [optional] diff --git a/rust/lance-namespace-reqwest-client/src/models/create_index_request.rs b/rust/lance-namespace-reqwest-client/src/models/create_index_request.rs index 032d90788..c5bb82491 100644 --- a/rust/lance-namespace-reqwest-client/src/models/create_index_request.rs +++ b/rust/lance-namespace-reqwest-client/src/models/create_index_request.rs @@ -28,21 +28,30 @@ pub struct CreateIndexRequest { /// Distance metric type for vector indexes #[serde(rename = "metric_type", skip_serializing_if = "Option::is_none")] pub metric_type: Option, - /// Number of partitions for IVF indexes - #[serde(rename = "num_partitions", skip_serializing_if = "Option::is_none")] - pub num_partitions: Option, - /// Number of sub-vectors for PQ indexes - #[serde(rename = "num_sub_vectors", skip_serializing_if = "Option::is_none")] - pub num_sub_vectors: Option, - /// Number of bits for scalar quantization - #[serde(rename = "num_bits", skip_serializing_if = "Option::is_none")] - pub num_bits: Option, - /// Maximum iterations for index building - #[serde(rename = "max_iterations", skip_serializing_if = "Option::is_none")] - pub max_iterations: Option, - /// Sample rate for index building - #[serde(rename = "sample_rate", skip_serializing_if = "Option::is_none")] - pub sample_rate: Option, + /// Optional FTS parameter for position tracking + #[serde(rename = "with_position", skip_serializing_if = "Option::is_none")] + pub with_position: Option, + /// Optional FTS parameter for base tokenizer + #[serde(rename = "base_tokenizer", skip_serializing_if = "Option::is_none")] + pub base_tokenizer: Option, + /// Optional FTS parameter for language + #[serde(rename = "language", skip_serializing_if = "Option::is_none")] + pub language: Option, + /// Optional FTS parameter for maximum token length + #[serde(rename = "max_token_length", skip_serializing_if = "Option::is_none")] + pub max_token_length: Option, + /// Optional FTS parameter for lowercase conversion + #[serde(rename = "lower_case", skip_serializing_if = "Option::is_none")] + pub lower_case: Option, + /// Optional FTS parameter for stemming + #[serde(rename = "stem", skip_serializing_if = "Option::is_none")] + pub stem: Option, + /// Optional FTS parameter for stop word removal + #[serde(rename = "remove_stop_words", skip_serializing_if = "Option::is_none")] + pub remove_stop_words: Option, + /// Optional FTS parameter for ASCII folding + #[serde(rename = "ascii_folding", skip_serializing_if = "Option::is_none")] + pub ascii_folding: Option, } impl CreateIndexRequest { @@ -53,11 +62,14 @@ impl CreateIndexRequest { column, index_type, metric_type: None, - num_partitions: None, - num_sub_vectors: None, - num_bits: None, - max_iterations: None, - sample_rate: None, + with_position: None, + base_tokenizer: None, + language: None, + max_token_length: None, + lower_case: None, + stem: None, + remove_stop_words: None, + ascii_folding: None, } } } diff --git a/rust/lance-namespace-reqwest-client/src/models/match_query.rs b/rust/lance-namespace-reqwest-client/src/models/match_query.rs index 8f684a19e..362bf08a5 100644 --- a/rust/lance-namespace-reqwest-client/src/models/match_query.rs +++ b/rust/lance-namespace-reqwest-client/src/models/match_query.rs @@ -15,8 +15,8 @@ use serde::{Deserialize, Serialize}; pub struct MatchQuery { #[serde(rename = "boost", skip_serializing_if = "Option::is_none")] pub boost: Option, - #[serde(rename = "column", default, with = "::serde_with::rust::double_option", skip_serializing_if = "Option::is_none")] - pub column: Option>, + #[serde(rename = "column", deserialize_with = "Option::deserialize")] + pub column: Option, #[serde(rename = "fuzziness", default, with = "::serde_with::rust::double_option", skip_serializing_if = "Option::is_none")] pub fuzziness: Option>, /// The maximum number of terms to expand for fuzzy matching. Default to 50. @@ -33,10 +33,10 @@ pub struct MatchQuery { } impl MatchQuery { - pub fn new(terms: String) -> MatchQuery { + pub fn new(column: Option, terms: String) -> MatchQuery { MatchQuery { boost: None, - column: None, + column, fuzziness: None, max_expansions: None, operator: None, From 0a622a5dd125df45c59136232a4dd2b2fc41e890 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Sun, 20 Jul 2025 16:18:38 -0700 Subject: [PATCH 10/10] Ease java API usage --- docs/src/user-guide/java-sdk.md | 47 ++--- .../lance/namespace/LanceRestNamespace.java | 10 ++ .../namespace/LanceRestNamespaceBuilder.java | 167 ++++++++++++++++++ .../namespace/test/BaseNamespaceTest.java | 37 ++-- 4 files changed, 207 insertions(+), 54 deletions(-) create mode 100644 java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespaceBuilder.java diff --git a/docs/src/user-guide/java-sdk.md b/docs/src/user-guide/java-sdk.md index 6d757709e..705882602 100644 --- a/docs/src/user-guide/java-sdk.md +++ b/docs/src/user-guide/java-sdk.md @@ -20,26 +20,16 @@ The artifact is available on [Maven Central](https://central.sonatype.com/artifa ### LanceDB Cloud -For LanceDB Cloud, use the following initialization approach: +For LanceDB Cloud, use the simplified builder API: ```java -import com.lancedb.lance.namespace.client.apache.ApiClient; import com.lancedb.lance.namespace.LanceRestNamespace; -import java.util.HashMap; -import java.util.Map; -// If your DB url is db://example-db, then your db here is example-db -String lancedbDatabase = "your_database_name"; -String lancedbApiKey = "your_lancedb_cloud_api_key"; - -Map config = new HashMap<>(); -config.put("headers.x-lancedb-database", lancedbDatabase); -config.put("headers.x-api-key", lancedbApiKey); - -ApiClient apiClient = new ApiClient(); -String baseUrl = String.format("https://%s.us-east-1.api.lancedb.com", lancedbDatabase); -apiClient.setBasePath(baseUrl); -LanceRestNamespace namespace = new LanceRestNamespace(apiClient, config); +// If your DB url is db://example-db, then your database here is example-db +LanceRestNamespace namespace = LanceRestNamespace.builder() + .apiKey("your_lancedb_cloud_api_key") + .database("your_database_name") + .build(); ``` ### LanceDB Enterprise @@ -47,19 +37,11 @@ LanceRestNamespace namespace = new LanceRestNamespace(apiClient, config); For Enterprise deployments, use your VPC endpoint: ```java -// Your top level folder under your cloud bucket, e.g. s3://your-bucket/your-top-dir/ -String lancedbDatabase = "your-top-dir"; -String lancedbApiKey = "your_lancedb_enterprise_api_key"; -// Your enterprise connection url -String lancedbHostOverride = "http://:80"; - -Map config = new HashMap<>(); -config.put("headers.x-lancedb-database", lancedbDatabase); -config.put("headers.x-api-key", lancedbApiKey); - -ApiClient apiClient = new ApiClient(); -apiClient.setBasePath(lancedbHostOverride); -LanceRestNamespace namespace = new LanceRestNamespace(apiClient, config); +LanceRestNamespace namespace = LanceRestNamespace.builder() + .apiKey("your_lancedb_enterprise_api_key") + .database("your-top-dir") // Your top level folder under your cloud bucket, e.g. s3://your-bucket/your-top-dir/ + .hostOverride("http://:80") + .build(); ``` ## Supported Endpoints @@ -327,8 +309,8 @@ CreateIndexRequest ftsIndexRequest = new CreateIndexRequest(); ftsIndexRequest.setName("documents"); ftsIndexRequest.setColumn("content"); ftsIndexRequest.setIndexType(CreateIndexRequest.IndexTypeEnum.FTS); -// Note: Set withPosition=true if you plan to use PhraseQuery -// ftsIndexRequest.setWithPosition(true); +// Set withPosition=true if you plan to use PhraseQuery +ftsIndexRequest.setWithPosition(true); CreateIndexResponse ftsResponse = namespace.createIndex(ftsIndexRequest); // Wait for index to be built @@ -395,6 +377,7 @@ StructuredFtsQuery structured = new StructuredFtsQuery(); FtsQuery ftsQuery = new FtsQuery(); // Boolean query: MUST contain "learning" AND (SHOULD contain "machine" OR "deep") +// Note: SHOULD clauses are optional when MUST clauses are present - they only affect ranking BooleanQuery boolQuery = new BooleanQuery(); // MUST clause: documents must contain "learning" @@ -459,6 +442,8 @@ phraseSearchQuery.setFullTextQuery(phraseFullText); byte[] phraseResults = namespace.queryTable(phraseSearchQuery); // Expected: Documents with "machine learning" or "machine [word] learning" +// Note: Phrase queries search for terms in the specified order. +// "learning machine" would NOT match this query. ``` !!! warning "PhraseQuery Requirements" diff --git a/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespace.java b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespace.java index 9087eb723..09653db5e 100644 --- a/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespace.java +++ b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespace.java @@ -83,6 +83,16 @@ public LanceRestNamespace(ApiClient client, Map config) { this.config = new RestConfig(config); } + /** + * Create a new builder for constructing LanceRestNamespace instances. This is the recommended way + * to create a LanceRestNamespace. + * + * @return A new LanceRestNamespaceBuilder + */ + public static LanceRestNamespaceBuilder builder() { + return LanceRestNamespaceBuilder.builder(); + } + @Override public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { try { diff --git a/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespaceBuilder.java b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespaceBuilder.java new file mode 100644 index 000000000..9291dbc0b --- /dev/null +++ b/java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceRestNamespaceBuilder.java @@ -0,0 +1,167 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace; + +import com.lancedb.lance.namespace.client.apache.ApiClient; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * Builder for creating LanceRestNamespace instances with simplified configuration. + * + *

Example usage for LanceDB Cloud: + * + * 

{@code
+ * LanceRestNamespace namespace = LanceRestNamespaceBuilder.builder()
+ *     .apiKey("your_api_key")
+ *     .database("your_database")
+ *     .region("us-east-1") // optional, defaults to us-east-1
+ *     .build();
+ * }
+ * + *

Example usage for LanceDB Enterprise (with host override): + * + * 

{@code
+ * LanceRestNamespace namespace = LanceRestNamespaceBuilder.builder()
+ *     .apiKey("your_api_key")
+ *     .database("your_database")
+ *     .hostOverride("http://your-vpc-endpoint:80")
+ *     .build();
+ * }
+ */ +public class LanceRestNamespaceBuilder { + private static final String DEFAULT_REGION = "us-east-1"; + private static final String CLOUD_URL_PATTERN = "https://%s.%s.api.lancedb.com"; + + private String apiKey; + private String database; + private Optional hostOverride = Optional.empty(); + private Optional region = Optional.empty(); + private Map additionalConfig = new HashMap<>(); + + private LanceRestNamespaceBuilder() {} + + /** + * Create a new builder instance. + * + * @return A new LanceRestNamespaceBuilder + */ + public static LanceRestNamespaceBuilder builder() { + return new LanceRestNamespaceBuilder(); + } + + /** + * Set the API key (required). + * + * @param apiKey The LanceDB API key + * @return This builder + */ + public LanceRestNamespaceBuilder apiKey(String apiKey) { + if (apiKey == null || apiKey.trim().isEmpty()) { + throw new IllegalArgumentException("API key cannot be null or empty"); + } + this.apiKey = apiKey; + return this; + } + + /** + * Set the database name (required). + * + * @param database The database name + * @return This builder + */ + public LanceRestNamespaceBuilder database(String database) { + if (database == null || database.trim().isEmpty()) { + throw new IllegalArgumentException("Database cannot be null or empty"); + } + this.database = database; + return this; + } + + /** + * Set a custom host override (optional). When set, this overrides the default LanceDB Cloud URL + * construction. Use this for LanceDB Enterprise deployments. + * + * @param hostOverride The complete base URL (e.g., "http://your-vpc-endpoint:80") + * @return This builder + */ + public LanceRestNamespaceBuilder hostOverride(String hostOverride) { + this.hostOverride = Optional.ofNullable(hostOverride); + return this; + } + + /** + * Set the region for LanceDB Cloud (optional). Defaults to "us-east-1" if not specified. This is + * ignored when hostOverride is set. + * + * @param region The AWS region (e.g., "us-east-1", "eu-west-1") + * @return This builder + */ + public LanceRestNamespaceBuilder region(String region) { + this.region = Optional.ofNullable(region); + return this; + } + + /** + * Add additional configuration parameters. + * + * @param key The configuration key + * @param value The configuration value + * @return This builder + */ + public LanceRestNamespaceBuilder config(String key, String value) { + this.additionalConfig.put(key, value); + return this; + } + + /** + * Build the LanceRestNamespace instance. + * + * @return A configured LanceRestNamespace + * @throws IllegalStateException if required parameters are missing + */ + public LanceRestNamespace build() { + // Validate required fields + if (apiKey == null) { + throw new IllegalStateException("API key is required"); + } + if (database == null) { + throw new IllegalStateException("Database is required"); + } + + // Build configuration map + Map config = new HashMap<>(additionalConfig); + config.put("headers.x-lancedb-database", database); + config.put("headers.x-api-key", apiKey); + + // Determine base URL + String baseUrl; + if (hostOverride.isPresent()) { + baseUrl = hostOverride.get(); + config.put("host_override", hostOverride.get()); + } else { + String effectiveRegion = region.orElse(DEFAULT_REGION); + baseUrl = String.format(CLOUD_URL_PATTERN, database, effectiveRegion); + config.put("region", effectiveRegion); + } + + // Create and configure ApiClient + ApiClient apiClient = new ApiClient(); + apiClient.setBasePath(baseUrl); + + return new LanceRestNamespace(apiClient, config); + } +} diff --git a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/BaseNamespaceTest.java b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/BaseNamespaceTest.java index 98925bd6f..f052ef1c2 100644 --- a/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/BaseNamespaceTest.java +++ b/java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/test/BaseNamespaceTest.java @@ -14,7 +14,7 @@ package com.lancedb.lance.namespace.test; import com.lancedb.lance.namespace.LanceRestNamespace; -import com.lancedb.lance.namespace.client.apache.ApiClient; +import com.lancedb.lance.namespace.LanceRestNamespaceBuilder; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -22,9 +22,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import java.util.HashMap; -import java.util.Map; - import static org.junit.jupiter.api.Assumptions.assumeTrue; /** @@ -90,35 +87,29 @@ protected void skipIfNotConfigured() { } /** - * Initialize the Lance REST client. + * Initialize the Lance REST client using the simplified builder API. * * @return Configured LanceRestNamespace instance */ private LanceRestNamespace initializeClient() { - Map config = new HashMap<>(); - config.put("headers.x-lancedb-database", DATABASE); - config.put("headers.x-api-key", API_KEY); + LanceRestNamespaceBuilder builder = + LanceRestNamespace.builder().apiKey(API_KEY).database(DATABASE); if (HOST_OVERRIDE != null) { - config.put("host_override", HOST_OVERRIDE); - } - if (REGION != null) { - config.put("region", REGION); + builder.hostOverride(HOST_OVERRIDE); + } else if (REGION != null) { + builder.region(REGION); } - ApiClient apiClient = new ApiClient(); - - // Set base URL based on configuration - String baseUrl; - if (HOST_OVERRIDE != null) { - baseUrl = HOST_OVERRIDE; - } else { - baseUrl = String.format("https://%s.%s.api.lancedb.com", DATABASE, REGION); - } - apiClient.setBasePath(baseUrl); + LanceRestNamespace namespace = builder.build(); + // Log the configuration for debugging + String baseUrl = + HOST_OVERRIDE != null + ? HOST_OVERRIDE + : String.format("https://%s.%s.api.lancedb.com", DATABASE, REGION); System.out.println("Initialized client with base URL: " + baseUrl); - return new LanceRestNamespace(apiClient, config); + return namespace; } }