diff --git a/CLAUDE.md b/CLAUDE.md index 1a86e217..5935399d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,204 +1,119 @@ -# Contributing to Lance Namespace +# CLAUDE.md -The Lance Namespace codebase is at [lance-format/lance-namespace](https://github.com/lance-format/lance-namespace). -This codebase contains: +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. -- The Lance Namespace specification -- The core `LanceNamespace` interface and generic connect functionality for all languages except Rust - (for Rust, these are located in the [lance-format/lance](https://github.com/lance-format/lance) repo) -- Generated clients and servers using OpenAPI generator +## What This Repo Is -This project should only be used to make spec and interface changes to Lance Namespace, -or to add new clients and servers to be generated based on community demand. -In general, we welcome more generated components to be added as long as -the contributor is willing to set up all the automations for generation and publication. +Lance Namespace specification, core interfaces, and OpenAPI-generated clients/servers. +The single source of truth is the OpenAPI spec at `docs/src/rest.yaml`. -For contributing changes to directory and REST namespaces, please go to the [lance](https://github.com/lance-format/lance) repo. +**Scope:** Only spec changes, interface changes, and new generated clients/servers belong here. +Implementation changes (directory/REST namespaces) go to [lance-format/lance](https://github.com/lance-format/lance). +Other namespace implementations go to [lance-format/lance-namespace-impls](https://github.com/lance-format/lance-namespace-impls). -For contributing changes to implementations other than the directory and REST namespace, -or for adding new namespace implementations, -please go to the [lance-namespace-impls](https://github.com/lance-format/lance-namespace-impls) repo. +## Build Commands -## Project Dependency +Requires [uv](https://docs.astral.sh/uv/getting-started/installation/). First run: `make sync` -This project contains the core Lance Namespace specification, interface and generated modules across all languages. -The dependency structure varies by language due to different build and distribution models. +| Command | Description | +|---------|-------------| +| `make lint` | Validate OpenAPI spec with openapi-spec-validator | +| `make gen` | Clean + codegen + lint all languages | +| `make build` | Full build: lint + docs + gen + build all languages | +| `make gen-` | Codegen one language: `gen-python`, `gen-java`, `gen-rust` | +| `make build-` | Build one language: `build-python`, `build-java`, `build-rust` | +| `make serve-docs` | Preview docs (auto-runs `gen-java` first) | -### Rust +Inside `java/` or `python/`, you can target specific modules: +`make gen-java-apache-client`, `make build-java-springboot-server`, etc. -For Rust, the interface module `lance-namespace` and implementations (`lance-namespace-impls` for REST and directory namespaces) -are located in the core [lance-format/lance](https://github.com/lance-format/lance) repository. -This is because Rust uses source code builds, and separating modules across repositories makes dependency management complicated. +### Running Tests -The dependency chain is: `lance-namespace` → `lance` → `lance-namespace-impls` - -### Other Languages (e.g. Python, Java) - -For Python, Java, and other languages, the core `LanceNamespace` interface and generic connect functionality -are maintained in **this repository** (e.g., `lance-namespace` for Python, `lance-namespace-core` for Java). -The core [lance-format/lance](https://github.com/lance-format/lance) repository then imports these modules. - -The reason for this import direction is that `lance-namespace-impls` (REST and directory namespace implementations) -are used in the Lance Python and Java bindings, and are exposed back through the corresponding language interfaces. -These language interfaces can also be imported dynamically without the need to have a dependency of the Lance core library bindings in those languages. - -### Other Implementations - -For namespace implementations other than directory and REST namespaces, -those are stored in the [lance-format/lance-namespace-impls](https://github.com/lance-format/lance-namespace-impls) repository, -with one implementation per language. - -### Dependency Diagram - -```mermaid -flowchart TB - subgraph this_repo["lance-namespace repo"] - spec["Spec & Generated Clients"] - py_core["Python: lance-namespace"] - java_core["Java: lance-namespace-core"] - end - - subgraph lance_repo["lance repo"] - subgraph rust_modules["Rust Modules"] - rs_ns["lance-namespace"] - rs_lance["lance"] - rs_impls["lance-namespace-impls
(dir, rest)"] - end - py_lance["Python: lance"] - java_lance["Java: lance"] - end - - subgraph impls_repo["namespace-impls repo"] - polaris["Apache Polaris"] ~~~ hive["Apache Hive"] ~~~ iceberg_rest["Apache Iceberg REST"] ~~~ unity["Unity Catalog"] ~~~ glue["AWS Glue"] - end - - %% Rust dependencies (source build) - rs_ns --> rs_lance - rs_lance --> rs_impls - - %% Python/Java dependencies - py_core --> py_lance - java_core --> java_lance - rs_impls -.-> py_lance - rs_impls -.-> java_lance - - %% Other implementations depend on core interfaces and lance bindings - py_core -.-> impls_repo - java_core -.-> impls_repo - py_lance -.-> impls_repo - java_lance -.-> impls_repo - - style this_repo fill:#1565c0,color:#fff - style lance_repo fill:#e65100,color:#fff - style impls_repo fill:#7b1fa2,color:#fff - style rust_modules fill:#ff8a65,color:#000 -``` - -## Repository structure - -This repository currently contains the following components: - -| Component | Language | Path | Description | -|-----------------------|----------|----------------------------------------|------------------------------------------------------------| -| Spec | | docs/src | Lance Namespace Specification | -| Python Core | Python | python/lance_namespace | Core LanceNamespace interface and connect functionality | -| Python UrlLib3 Client | Python | python/lance_namespace_urllib3_client | Generated Python urllib3 client for Lance REST Namespace | -| Java Core | Java | java/lance-namespace-core | Core LanceNamespace interface and connect functionality | -| Java Apache Client | Java | java/lance-namespace-apache-client | Generated Java Apache HTTP client for Lance REST Namespace | -| Java SpringBoot Server| Java | java/lance-namespace-springboot-server | Generated Java SpringBoot server for Lance REST Namespace | -| Rust Reqwest Client | Rust | rust/lance-namespace-reqwest-client | Generated Rust reqwest client for Lance REST Namespace | - - -## Install uv +```bash +# Python +cd python/lance_namespace && uv sync && uv run pytest +cd python/lance_namespace_urllib3_client && uv sync && uv run pytest -We use [uv](https://docs.astral.sh/uv/getting-started/installation/) for development. -Make sure it is installed, and run: +# Java (checkstyle + spotless + maven build with tests) +cd java && make check # style checks only +cd java && make build # full build including tests -```bash -make sync +# Rust +cd rust && cargo test --all-features ``` -## Lint - -To ensure the OpenAPI definition is valid, you can use the lint command to check it. +### Java Style Checks +Java uses Spotless (formatting) and Checkstyle (linting). The `java/Makefile` `check` target +runs both. These are enforced in CI. Fix formatting issues with: ```bash -make lint +cd java && mvn spotless:apply ``` -## Build - -There are 3 commands that is available at top level as well as inside each language folder: +## Generated vs Hand-Written Code -- `make clean`: remove all codegen modules -- `make gen`: codegen and lint all modules (depends on `clean`) -- `make build`: build all modules (depends on `gen`) +**Never manually edit generated code.** CI (`spec.yml`) verifies that running `make clean && make gen` +produces no diff — any manual edits to generated files will be rejected. -You can also run `make -` to only run the command in the specific language, for example: +### Hand-written (edit these): +- `docs/src/rest.yaml` — OpenAPI spec, the single source of truth +- `python/lance_namespace/` — Python core interface, connect factory, error hierarchy +- `java/lance-namespace-core/` — Java core interface, connect factory, errors +- `java/lance-namespace-core-async/` — Java async wrapper around core +- `java/openapi-templates/` — Custom Mustache templates for Java codegen -- `make gen-python`: codegen and lint all Python modules -- `make build-rust`: build all Rust modules +### Generated (do not edit): +- `python/lance_namespace_urllib3_client/` — Python HTTP client + all model classes +- `java/lance-namespace-apache-client/` — Java Apache HttpClient implementation +- `java/lance-namespace-async-client/` — Java native async HttpClient implementation +- `java/lance-namespace-springboot-server/` — Spring Boot server skeleton +- `rust/lance-namespace-reqwest-client/` — Rust reqwest client -You can also run `make --` inside a language folder to run the command against a specific module, for example: +Codegen uses `openapi-generator-cli` (v7.12.0 via uv). Language-specific ignore files +(e.g., `.apache-client-ignore`) control which generated artifacts are committed. -- `make gen-rust-reqwest-client`: codegen and lint the Rust reqwest client module -- `make build-java-springboot-server`: build the Java Spring Boot server module +## Architecture -## Documentation +### Plugin/Registry Pattern -### Setup +Both Python and Java use a plugin system where implementations are discovered at runtime: -The documentation website is built using [mkdocs-material](https://pypi.org/project/mkdocs-material). -Start the server with: - -```shell -make serve-docs -``` - -### Generated Model Documentation - -The operation request and response model documentation is generated from the Java Apache Client. -When building or serving docs, the Java client must be generated first to produce the model Markdown files, -which are then copied to `docs/src/operations/models/`. - -This happens automatically when running: - -```shell -make build-docs # or make serve-docs -``` +**Python** (`lance_namespace/__init__.py`): +- `connect(impl, properties)` — factory that resolves an implementation name +- `register_namespace_impl(name, class_path)` — register external implementations +- Resolution: native aliases ("dir", "rest") → registered impls → full module.Class path +- Uses `importlib.import_module()` for dynamic loading -These commands depend on `gen-java` to ensure the Java client docs are up-to-date before building the documentation. +**Java** (`LanceNamespace.java`): +- `LanceNamespace.connect(impl, properties, allocator)` — static factory +- `registerNamespaceImpl(name, className)` / `unregisterNamespaceImpl(name)` +- Resolution: `NATIVE_IMPLS` map → `REGISTERED_IMPLS` concurrent map → full class name +- Uses reflection with no-arg constructor + `initialize()` call +- Requires Apache Arrow `BufferAllocator` parameter -### Understanding the Build Process +### Error System -The contents in `lance-namespace/docs` are for the ease of contributors to edit and preview. -After code merge, the contents are added to the -[main Lance documentation](https://github.com/lance-format/lance/tree/main/docs) -during the Lance doc CI build time, and is presented in the Lance website under -[Lance Namespace Spec](https://lance.org/lance/format/namespace). +Consistent error codes (0-21) across all languages in `ErrorCode` enum/class. +Each code has a corresponding exception class. Factory function `from_error_code()` maps codes to exceptions. -## Release Process +### API Operations -This section describes the CI/CD workflows for automated version management, releases, and publishing. +The REST spec defines 40+ endpoints under `/v1/` organized as: +- **Namespace ops:** create, list, describe, drop, exists +- **Table ops:** CRUD, schema mutations, versioning, indexing, tags, query/insert/merge +- **Transaction ops:** describe, alter +- **Batch ops:** batch version create, batch commit (atomic multi-table) -### Version Scheme +All operations are default methods on `LanceNamespace` that throw `UnsupportedOperationError`, +allowing implementations to opt into only the operations they support. -- **Stable releases:** `X.Y.Z` (e.g., 1.2.3) -- **Preview releases:** `X.Y.Z-beta.N` (e.g., 1.2.3-beta.1) +### Documentation Build -### Creating a Release +Model docs are generated from the Java Apache Client's Javadoc and copied to `docs/src/operations/models/`. +This is why `build-docs` and `serve-docs` depend on `gen-java`. -1. **Create Release Draft** - - Go to Actions → "Create Release" - - Select parameters: - - Release type (major/minor/patch) - - Release channel (stable/preview) - - Dry run (test without pushing) - - Run workflow (creates a draft release) +## Dependency Structure -2. **Review and Publish** - - Go to the [Releases page](../../releases) to review the draft - - Edit release notes if needed - - Click "Publish release" to: - - For stable releases: Trigger automatic publishing for Java, Python, Rust - - For preview releases: Create a beta release (not published) +- **Rust:** Interface lives in the [lance](https://github.com/lance-format/lance) repo, not here. Only the generated reqwest client is here. +- **Python/Java:** Core interfaces live here; implementations are in the lance repo and consume these interfaces. +- The Python core package re-exports all model types from the generated urllib3 client, so downstream code only needs to import `lance_namespace`. diff --git a/java/Makefile b/java/Makefile index bff81d2b..2da413f3 100644 --- a/java/Makefile +++ b/java/Makefile @@ -133,11 +133,24 @@ check-apache-client: check-springboot-server: ./mvnw checkstyle:check spotless:check -pl lance-namespace-springboot-server -am +# lance-namespace-base module (hand-written, no codegen) +.PHONY: lint-base +lint-base: gen-apache-client + ./mvnw spotless:apply -pl lance-namespace-base -am + +.PHONY: build-base +build-base: build-apache-client build-core lint-base + ./mvnw install -pl lance-namespace-base -am + +.PHONY: check-base +check-base: + ./mvnw checkstyle:check spotless:check -pl lance-namespace-base -am + .PHONY: check -check: check-apache-client check-async-client check-springboot-server check-core check-core-async +check: check-apache-client check-async-client check-springboot-server check-core check-core-async check-base .PHONY: lint -lint: lint-apache-client lint-async-client lint-springboot-server lint-core lint-core-async +lint: lint-apache-client lint-async-client lint-springboot-server lint-core lint-core-async lint-base .PHONY: build -build: build-apache-client build-async-client build-springboot-server build-core build-core-async +build: build-apache-client build-async-client build-springboot-server build-core build-core-async build-base diff --git a/java/lance-namespace-base/pom.xml b/java/lance-namespace-base/pom.xml new file mode 100644 index 00000000..8c3b624d --- /dev/null +++ b/java/lance-namespace-base/pom.xml @@ -0,0 +1,101 @@ + + + 4.0.0 + + + org.lance + lance-namespace-root + 0.6.1 + + + lance-namespace-base + jar + + lance-namespace-base + Base LanceNamespace implementation using Lance Java SDK + + + 5.0.0-beta.1 + + 18.3.0 + + + + + + org.lance + lance-namespace-core + ${project.version} + + + + + org.lance + lance-namespace-apache-client + ${project.version} + + + + + org.lance + lance-core + ${lance-core.version} + + + + + org.apache.arrow + arrow-vector + ${lance-arrow.version} + + + org.apache.arrow + arrow-memory-core + ${lance-arrow.version} + + + org.apache.arrow + arrow-c-data + ${lance-arrow.version} + + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.apache.arrow + arrow-memory-netty + ${lance-arrow.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.10.1 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + + diff --git a/java/lance-namespace-base/src/main/java/org/lance/namespace/base/BaseLanceNamespace.java b/java/lance-namespace-base/src/main/java/org/lance/namespace/base/BaseLanceNamespace.java new file mode 100644 index 00000000..77ab2af7 --- /dev/null +++ b/java/lance-namespace-base/src/main/java/org/lance/namespace/base/BaseLanceNamespace.java @@ -0,0 +1,1235 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.namespace.base; + +import org.lance.Dataset; +import org.lance.ReadOptions; +import org.lance.Tag; +import org.lance.Version; +import org.lance.WriteParams; +import org.lance.index.DistanceType; +import org.lance.index.Index; +import org.lance.index.IndexDescription; +import org.lance.index.IndexOptions; +import org.lance.index.IndexParams; +import org.lance.index.IndexType; +import org.lance.index.scalar.ScalarIndexParams; +import org.lance.ipc.FullTextQuery; +import org.lance.ipc.LanceScanner; +import org.lance.ipc.Query; +import org.lance.ipc.ScanOptions; +import org.lance.merge.MergeInsertParams; +import org.lance.namespace.LanceNamespace; +import org.lance.namespace.errors.InternalException; +import org.lance.namespace.errors.InvalidInputException; +import org.lance.namespace.errors.LanceNamespaceException; +import org.lance.namespace.errors.NamespaceAlreadyExistsException; +import org.lance.namespace.errors.NamespaceNotEmptyException; +import org.lance.namespace.errors.NamespaceNotFoundException; +import org.lance.namespace.errors.TableAlreadyExistsException; +import org.lance.namespace.errors.TableIndexNotFoundException; +import org.lance.namespace.errors.TableNotFoundException; +import org.lance.namespace.errors.TableTagNotFoundException; +import org.lance.namespace.errors.TableVersionNotFoundException; +import org.lance.namespace.model.*; +import org.lance.schema.ColumnAlteration; +import org.lance.schema.LanceField; +import org.lance.schema.LanceSchema; +import org.lance.schema.SqlExpressions; + +import org.apache.arrow.c.ArrowArrayStream; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.pojo.Schema; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Base implementation of {@link LanceNamespace} using the Lance Java SDK. + * + *

This class provides a filesystem-directory-based namespace implementation that delegates table + * operations to the Lance Java SDK ({@link Dataset}, {@link LanceScanner}, etc.) rather than JNI + * namespace calls. + * + *

Namespace hierarchy maps to filesystem directories: namespace path {@code ["a", "b"]} resolves + * to {@code {root}/a/b/}. Tables within a namespace resolve to {@code {root}/a/b/tableName.lance}. + * + *

Configuration properties: + * + *

    + *
  • {@code root} (required): Root directory path + *
  • {@code storage.*} (optional): Storage options passed to Dataset operations + *
+ */ +public class BaseLanceNamespace implements LanceNamespace, Closeable { + + private static final String LANCE_EXTENSION = ".lance"; + private static final String STORAGE_PREFIX = "storage."; + + private BufferAllocator allocator; + private String root; + private Map storageOptions; + + public BaseLanceNamespace() {} + + @Override + public void initialize(Map configProperties, BufferAllocator allocator) { + this.allocator = allocator; + this.root = configProperties.get("root"); + if (this.root == null || this.root.isEmpty()) { + throw new InvalidInputException("Property 'root' is required"); + } + this.storageOptions = new HashMap<>(); + for (Map.Entry entry : configProperties.entrySet()) { + if (entry.getKey().startsWith(STORAGE_PREFIX)) { + storageOptions.put(entry.getKey().substring(STORAGE_PREFIX.length()), entry.getValue()); + } + } + } + + @Override + public String namespaceId() { + return "BaseLanceNamespace{root=" + root + "}"; + } + + @Override + public void close() { + // No native resources to release + } + + // ========== Namespace Operations ========== + + @Override + public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { + Path nsPath = resolveNamespacePath(request.getId()); + if (Files.exists(nsPath)) { + throw new NamespaceAlreadyExistsException( + "Namespace already exists: " + String.join("/", request.getId())); + } + try { + Files.createDirectories(nsPath); + } catch (IOException e) { + throw new InternalException("Failed to create namespace: " + e.getMessage()); + } + return new CreateNamespaceResponse().properties(new HashMap<>()); + } + + @Override + public ListNamespacesResponse listNamespaces(ListNamespacesRequest request) { + List id = request.getId(); + Path nsPath = (id == null || id.isEmpty()) ? Paths.get(root) : resolveNamespacePath(id); + ensureNamespaceExists(nsPath, id); + + List namespaces = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(nsPath)) { + for (Path entry : stream) { + if (Files.isDirectory(entry) && !entry.getFileName().toString().endsWith(LANCE_EXTENSION)) { + namespaces.add(entry.getFileName().toString()); + } + } + } catch (IOException e) { + throw new InternalException("Failed to list namespaces: " + e.getMessage()); + } + Collections.sort(namespaces); + String[] nextToken = new String[1]; + namespaces = paginateList(namespaces, request.getLimit(), request.getPageToken(), nextToken); + ListNamespacesResponse response = + new ListNamespacesResponse().namespaces(new LinkedHashSet<>(namespaces)); + if (nextToken[0] != null) { + response.setPageToken(nextToken[0]); + } + return response; + } + + @Override + public DescribeNamespaceResponse describeNamespace(DescribeNamespaceRequest request) { + Path nsPath = resolveNamespacePath(request.getId()); + ensureNamespaceExists(nsPath, request.getId()); + return new DescribeNamespaceResponse().properties(new HashMap<>()); + } + + @Override + public DropNamespaceResponse dropNamespace(DropNamespaceRequest request) { + Path nsPath = resolveNamespacePath(request.getId()); + ensureNamespaceExists(nsPath, request.getId()); + + try (DirectoryStream stream = Files.newDirectoryStream(nsPath)) { + if (stream.iterator().hasNext()) { + throw new NamespaceNotEmptyException( + "Namespace is not empty: " + String.join("/", request.getId())); + } + } catch (LanceNamespaceException e) { + throw e; + } catch (IOException e) { + throw new InternalException("Failed to check namespace: " + e.getMessage()); + } + + try { + Files.delete(nsPath); + } catch (IOException e) { + throw new InternalException("Failed to drop namespace: " + e.getMessage()); + } + return new DropNamespaceResponse(); + } + + @Override + public void namespaceExists(NamespaceExistsRequest request) { + Path nsPath = resolveNamespacePath(request.getId()); + ensureNamespaceExists(nsPath, request.getId()); + } + + // ========== Table Operations ========== + + @Override + public CreateTableResponse createTable(CreateTableRequest request, byte[] requestData) { + String tableUri = resolveTableUri(request.getId()); + if (Files.exists(Paths.get(tableUri))) { + throw new TableAlreadyExistsException("Table already exists: " + tableName(request.getId())); + } + ensureParentNamespaceExists(request.getId()); + + try (ArrowArrayStream stream = ipcBytesToStream(requestData); + Dataset dataset = + Dataset.write().allocator(allocator).stream(stream) + .uri(tableUri) + .storageOptions(storageOptions) + .mode(WriteParams.WriteMode.CREATE) + .execute()) { + return new CreateTableResponse().location(tableUri).version(dataset.version()); + } + } + + @Override + public DeclareTableResponse declareTable(DeclareTableRequest request) { + String tableUri = resolveTableUri(request.getId()); + if (Files.exists(Paths.get(tableUri))) { + throw new TableAlreadyExistsException("Table already exists: " + tableName(request.getId())); + } + ensureParentNamespaceExists(request.getId()); + + try { + Files.createDirectories(Paths.get(tableUri)); + } catch (IOException e) { + throw new InternalException("Failed to declare table: " + e.getMessage()); + } + return new DeclareTableResponse().location(tableUri); + } + + @Override + public ListTablesResponse listTables(ListTablesRequest request) { + List id = request.getId(); + Path nsPath = (id == null || id.isEmpty()) ? Paths.get(root) : resolveNamespacePath(id); + ensureNamespaceExists(nsPath, id); + + List tables = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(nsPath)) { + for (Path entry : stream) { + String name = entry.getFileName().toString(); + if (Files.isDirectory(entry) && name.endsWith(LANCE_EXTENSION)) { + tables.add(name.substring(0, name.length() - LANCE_EXTENSION.length())); + } + } + } catch (IOException e) { + throw new InternalException("Failed to list tables: " + e.getMessage()); + } + Collections.sort(tables); + String[] nextToken = new String[1]; + tables = paginateList(tables, request.getLimit(), request.getPageToken(), nextToken); + ListTablesResponse response = new ListTablesResponse().tables(new LinkedHashSet<>(tables)); + if (nextToken[0] != null) { + response.setPageToken(nextToken[0]); + } + return response; + } + + @Override + public DescribeTableResponse describeTable(DescribeTableRequest request) { + String tableUri = resolveTableUri(request.getId()); + ensureTableExists(request.getId()); + + try (Dataset dataset = openDataset(request.getId())) { + Schema schema = dataset.getSchema(); + return new DescribeTableResponse() + .location(tableUri) + .version(dataset.version()) + .schema(convertArrowSchemaToJson(schema)); + } + } + + @Override + public void tableExists(TableExistsRequest request) { + ensureTableExists(request.getId()); + } + + @Override + public DropTableResponse dropTable(DropTableRequest request) { + Path tablePath = Paths.get(resolveTableUri(request.getId())); + ensureTableExists(request.getId()); + + try { + deleteRecursive(tablePath); + } catch (IOException e) { + throw new InternalException("Failed to drop table: " + e.getMessage()); + } + return new DropTableResponse(); + } + + @Override + public RegisterTableResponse registerTable(RegisterTableRequest request) { + String location = request.getLocation(); + if (location == null || location.isEmpty()) { + throw new InvalidInputException("Location is required for registerTable"); + } + List tableId = request.getId(); + String tableUri = resolveTableUri(tableId); + if (Files.exists(Paths.get(tableUri))) { + throw new TableAlreadyExistsException("Table already exists: " + tableName(tableId)); + } + ensureParentNamespaceExists(tableId); + + // Verify the location is a valid Lance dataset by opening it + try (Dataset dataset = + Dataset.open( + allocator, + location, + new ReadOptions.Builder().setStorageOptions(storageOptions).build())) { + // Dataset is valid -- create a symlink to register it under this namespace + Path target = Paths.get(location); + Path link = Paths.get(tableUri); + try { + Files.createSymbolicLink(link, target); + } catch (IOException e) { + throw new InternalException("Failed to register table: " + e.getMessage()); + } + return new RegisterTableResponse().location(tableUri); + } catch (LanceNamespaceException e) { + throw e; + } catch (RuntimeException e) { + throw new InvalidInputException( + "Location is not a valid Lance dataset: " + location + " (" + e.getMessage() + ")"); + } + } + + @Override + public DeregisterTableResponse deregisterTable(DeregisterTableRequest request) { + // For filesystem-based namespace, deregister is the same as drop + dropTable(new DropTableRequest().id(request.getId())); + return new DeregisterTableResponse(); + } + + @Override + public Long countTableRows(CountTableRowsRequest request) { + ensureTableExists(request.getId()); + try (Dataset dataset = openDataset(request.getId())) { + String predicate = request.getPredicate(); + if (predicate != null && !predicate.isEmpty()) { + return dataset.countRows(predicate); + } + return dataset.countRows(); + } + } + + // ========== Data Operations ========== + + @Override + public InsertIntoTableResponse insertIntoTable( + InsertIntoTableRequest request, byte[] requestData) { + ensureTableExists(request.getId()); + String tableUri = resolveTableUri(request.getId()); + + try (ArrowArrayStream stream = ipcBytesToStream(requestData); + Dataset dataset = + Dataset.write().allocator(allocator).stream(stream) + .uri(tableUri) + .storageOptions(storageOptions) + .mode(WriteParams.WriteMode.APPEND) + .execute()) { + return new InsertIntoTableResponse(); + } + } + + @Override + public MergeInsertIntoTableResponse mergeInsertIntoTable( + MergeInsertIntoTableRequest request, byte[] requestData) { + ensureTableExists(request.getId()); + + MergeInsertParams params = buildMergeInsertParams(request); + + try (Dataset dataset = openDataset(request.getId()); + ArrowArrayStream stream = ipcBytesToStream(requestData)) { + dataset.mergeInsert(params, stream); + return new MergeInsertIntoTableResponse().version(dataset.version()); + } + } + + @Override + public DeleteFromTableResponse deleteFromTable(DeleteFromTableRequest request) { + ensureTableExists(request.getId()); + String predicate = request.getPredicate(); + if (predicate == null || predicate.isEmpty()) { + throw new InvalidInputException("Filter predicate is required for delete"); + } + + try (Dataset dataset = openDataset(request.getId())) { + dataset.delete(predicate); + return new DeleteFromTableResponse(); + } + } + + @Override + public byte[] queryTable(QueryTableRequest request) { + ensureTableExists(request.getId()); + + Long version = request.getVersion(); + try (Dataset dataset = + version != null ? openDataset(request.getId(), version) : openDataset(request.getId())) { + + ScanOptions scanOptions = buildScanOptions(request); + LanceScanner scanner = dataset.newScan(scanOptions); + try { + return scannerToIpcBytes(scanner); + } finally { + closeQuietly(scanner); + } + } + } + + // ========== Index Operations ========== + + @Override + public CreateTableIndexResponse createTableIndex(CreateTableIndexRequest request) { + ensureTableExists(request.getId()); + + try (Dataset dataset = openDataset(request.getId())) { + IndexOptions options = buildIndexOptions(request, false); + Index index = dataset.createIndex(options); + return new CreateTableIndexResponse(); + } + } + + @Override + public CreateTableScalarIndexResponse createTableScalarIndex(CreateTableIndexRequest request) { + ensureTableExists(request.getId()); + + try (Dataset dataset = openDataset(request.getId())) { + IndexOptions options = buildIndexOptions(request, true); + dataset.createIndex(options); + return new CreateTableScalarIndexResponse(); + } + } + + @Override + public ListTableIndicesResponse listTableIndices(ListTableIndicesRequest request) { + ensureTableExists(request.getId()); + + try (Dataset dataset = openDataset(request.getId())) { + Map fieldIdToName = buildFieldIdToNameMap(dataset); + List indices = dataset.getIndexes(); + List indexContents = new ArrayList<>(); + for (Index index : indices) { + IndexContent content = new IndexContent(); + content.setIndexName(index.name()); + content.setIndexUuid(index.uuid() != null ? index.uuid().toString() : ""); + content.setColumns( + index.fields().stream() + .map(fieldId -> fieldIdToName.getOrDefault(fieldId, String.valueOf(fieldId))) + .collect(Collectors.toList())); + content.setStatus("READY"); + indexContents.add(content); + } + Integer limit = request.getLimit(); + if (limit != null && limit > 0 && indexContents.size() > limit) { + indexContents = indexContents.subList(0, limit); + } + return new ListTableIndicesResponse().indexes(indexContents); + } + } + + @Override + public DescribeTableIndexStatsResponse describeTableIndexStats( + DescribeTableIndexStatsRequest request, String indexName) { + ensureTableExists(request.getId()); + + try (Dataset dataset = openDataset(request.getId())) { + List descriptions = dataset.describeIndices(); + for (IndexDescription desc : descriptions) { + if (indexName.equals(desc.getName())) { + long totalRows = dataset.countRows(); + long indexedRows = desc.getRowsIndexed(); + long unindexedRows = Math.max(0, totalRows - indexedRows); + DescribeTableIndexStatsResponse response = new DescribeTableIndexStatsResponse(); + response.setIndexType(desc.getIndexType()); + response.setNumIndexedRows(indexedRows); + response.setNumUnindexedRows(unindexedRows); + response.setNumIndices(desc.getSegments().size()); + return response; + } + } + throw new TableIndexNotFoundException("Index not found: " + indexName); + } + } + + @Override + public DropTableIndexResponse dropTableIndex(DropTableIndexRequest request, String indexName) { + ensureTableExists(request.getId()); + + try (Dataset dataset = openDataset(request.getId())) { + dataset.dropIndex(indexName); + return new DropTableIndexResponse(); + } + } + + // ========== Schema Operations ========== + + @Override + public AlterTableAddColumnsResponse alterTableAddColumns(AlterTableAddColumnsRequest request) { + ensureTableExists(request.getId()); + + List transforms = request.getNewColumns(); + if (transforms == null || transforms.isEmpty()) { + throw new InvalidInputException("At least one column transform is required"); + } + + SqlExpressions.Builder builder = new SqlExpressions.Builder(); + for (NewColumnTransform transform : transforms) { + builder.withExpression(transform.getName(), transform.getExpression()); + } + + try (Dataset dataset = openDataset(request.getId())) { + dataset.addColumns(builder.build(), Optional.empty()); + return new AlterTableAddColumnsResponse(); + } + } + + @Override + public AlterTableAlterColumnsResponse alterTableAlterColumns( + AlterTableAlterColumnsRequest request) { + ensureTableExists(request.getId()); + + List entries = request.getAlterations(); + if (entries == null || entries.isEmpty()) { + throw new InvalidInputException("At least one column alteration is required"); + } + + List alterations = new ArrayList<>(); + for (AlterColumnsEntry entry : entries) { + ColumnAlteration.Builder builder = new ColumnAlteration.Builder(entry.getPath()); + if (entry.getRename() != null) { + builder.rename(entry.getRename()); + } + if (entry.getNullable() != null) { + builder.nullable(entry.getNullable()); + } + alterations.add(builder.build()); + } + + try (Dataset dataset = openDataset(request.getId())) { + dataset.alterColumns(alterations); + return new AlterTableAlterColumnsResponse(); + } + } + + @Override + public AlterTableDropColumnsResponse alterTableDropColumns(AlterTableDropColumnsRequest request) { + ensureTableExists(request.getId()); + + List columns = request.getColumns(); + if (columns == null || columns.isEmpty()) { + throw new InvalidInputException("At least one column name is required"); + } + + try (Dataset dataset = openDataset(request.getId())) { + dataset.dropColumns(columns); + return new AlterTableDropColumnsResponse(); + } + } + + @Override + public GetTableStatsResponse getTableStats(GetTableStatsRequest request) { + ensureTableExists(request.getId()); + + try (Dataset dataset = openDataset(request.getId())) { + long rowCount = dataset.countRows(); + return new GetTableStatsResponse().numRows(rowCount); + } + } + + // ========== Version Operations ========== + + @Override + public ListTableVersionsResponse listTableVersions(ListTableVersionsRequest request) { + ensureTableExists(request.getId()); + + try (Dataset dataset = openDataset(request.getId())) { + List versions = dataset.listVersions(); + List tableVersions = new ArrayList<>(); + for (Version version : versions) { + tableVersions.add(toTableVersion(version)); + } + Integer limit = request.getLimit(); + if (limit != null && limit > 0 && tableVersions.size() > limit) { + tableVersions = tableVersions.subList(0, limit); + } + return new ListTableVersionsResponse().versions(tableVersions); + } + } + + @Override + public DescribeTableVersionResponse describeTableVersion(DescribeTableVersionRequest request) { + ensureTableExists(request.getId()); + + Long versionNum = request.getVersion(); + if (versionNum == null) { + throw new InvalidInputException("Version is required for describeTableVersion"); + } + + try (Dataset dataset = openDataset(request.getId())) { + List versions = dataset.listVersions(); + for (Version version : versions) { + if (version.getId() == versionNum.longValue()) { + return new DescribeTableVersionResponse().version(toTableVersion(version)); + } + } + throw new TableVersionNotFoundException("Version not found: " + versionNum); + } + } + + @Override + public RestoreTableResponse restoreTable(RestoreTableRequest request) { + ensureTableExists(request.getId()); + + Long version = request.getVersion(); + if (version == null) { + throw new InvalidInputException("Version is required for restore"); + } + + try (Dataset dataset = openDataset(request.getId(), version)) { + dataset.restore(); + return new RestoreTableResponse(); + } + } + + @Override + public RenameTableResponse renameTable(RenameTableRequest request) { + ensureTableExists(request.getId()); + + String newName = request.getNewTableName(); + if (newName == null || newName.isEmpty()) { + throw new InvalidInputException("New table name is required"); + } + + Path sourcePath = Paths.get(resolveTableUri(request.getId())); + List namespacePath = namespacePath(request.getId()); + List newId = new ArrayList<>(namespacePath); + newId.add(newName); + Path targetPath = Paths.get(resolveTableUri(newId)); + + if (Files.exists(targetPath)) { + throw new TableAlreadyExistsException("Table already exists: " + newName); + } + + try { + Files.move(sourcePath, targetPath); + } catch (IOException e) { + throw new InternalException("Failed to rename table: " + e.getMessage()); + } + return new RenameTableResponse(); + } + + @Override + public ListTablesResponse listAllTables(ListTablesRequest request) { + List tables = new ArrayList<>(); + collectTablesRecursive(Paths.get(root), "", tables); + Collections.sort(tables); + String[] nextToken = new String[1]; + tables = paginateList(tables, request.getLimit(), request.getPageToken(), nextToken); + ListTablesResponse response = new ListTablesResponse().tables(new LinkedHashSet<>(tables)); + if (nextToken[0] != null) { + response.setPageToken(nextToken[0]); + } + return response; + } + + // ========== Tag Operations ========== + + @Override + public CreateTableTagResponse createTableTag(CreateTableTagRequest request) { + ensureTableExists(request.getId()); + + String tagName = request.getTag(); + Long version = request.getVersion(); + if (tagName == null || version == null) { + throw new InvalidInputException("Tag name and version are required"); + } + + try (Dataset dataset = openDataset(request.getId())) { + dataset.tags().create(tagName, version); + return new CreateTableTagResponse(); + } + } + + @Override + public ListTableTagsResponse listTableTags(ListTableTagsRequest request) { + ensureTableExists(request.getId()); + + try (Dataset dataset = openDataset(request.getId())) { + List tags = dataset.tags().list(); + Integer limit = request.getLimit(); + Map tagMap = new LinkedHashMap<>(); + int count = 0; + for (Tag tag : tags) { + if (limit != null && limit > 0 && count >= limit) { + break; + } + TagContents tc = new TagContents(); + tc.setVersion(tag.getVersion()); + tc.setManifestSize((long) tag.getManifestSize()); + tag.getBranch().ifPresent(tc::setBranch); + tagMap.put(tag.getName(), tc); + count++; + } + return new ListTableTagsResponse().tags(tagMap); + } + } + + @Override + public GetTableTagVersionResponse getTableTagVersion(GetTableTagVersionRequest request) { + ensureTableExists(request.getId()); + + String tagName = request.getTag(); + if (tagName == null) { + throw new InvalidInputException("Tag name is required"); + } + + return withTagNotFoundHandling( + tagName, + () -> { + try (Dataset dataset = openDataset(request.getId())) { + long version = dataset.tags().getVersion(tagName); + return new GetTableTagVersionResponse().version(version); + } + }); + } + + @Override + public DeleteTableTagResponse deleteTableTag(DeleteTableTagRequest request) { + ensureTableExists(request.getId()); + + String tagName = request.getTag(); + if (tagName == null) { + throw new InvalidInputException("Tag name is required"); + } + + return withTagNotFoundHandling( + tagName, + () -> { + try (Dataset dataset = openDataset(request.getId())) { + dataset.tags().delete(tagName); + return new DeleteTableTagResponse(); + } + }); + } + + @Override + public UpdateTableTagResponse updateTableTag(UpdateTableTagRequest request) { + ensureTableExists(request.getId()); + + String tagName = request.getTag(); + Long version = request.getVersion(); + if (tagName == null || version == null) { + throw new InvalidInputException("Tag name and version are required"); + } + + return withTagNotFoundHandling( + tagName, + () -> { + try (Dataset dataset = openDataset(request.getId())) { + dataset.tags().update(tagName, version); + return new UpdateTableTagResponse(); + } + }); + } + + // ========== Protected Helper Methods ========== + + protected Dataset openDataset(List tableId) { + String tableUri = resolveTableUri(tableId); + ReadOptions options = new ReadOptions.Builder().setStorageOptions(storageOptions).build(); + return Dataset.open(allocator, tableUri, options); + } + + protected Dataset openDataset(List tableId, long version) { + String tableUri = resolveTableUri(tableId); + ReadOptions options = + new ReadOptions.Builder().setStorageOptions(storageOptions).setVersion(version).build(); + return Dataset.open(allocator, tableUri, options); + } + + protected String resolveTableUri(List tableId) { + if (tableId == null || tableId.isEmpty()) { + throw new InvalidInputException("Table ID is required"); + } + StringBuilder sb = new StringBuilder(root); + for (String part : tableId) { + sb.append("/").append(part); + } + sb.append(LANCE_EXTENSION); + return sb.toString(); + } + + protected Path resolveNamespacePath(List namespaceId) { + if (namespaceId == null || namespaceId.isEmpty()) { + return Paths.get(root); + } + StringBuilder sb = new StringBuilder(root); + for (String part : namespaceId) { + sb.append("/").append(part); + } + return Paths.get(sb.toString()); + } + + protected ArrowArrayStream ipcBytesToStream(byte[] data) { + ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator); + try { + // exportArrayStream takes ownership of the reader -- it will be closed when the stream is + // consumed. Do NOT close the reader here or wrap it in try-with-resources. + ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream(data), allocator); + Data.exportArrayStream(allocator, reader, stream); + } catch (Exception e) { + stream.close(); + throw new InvalidInputException("Failed to parse Arrow IPC data: " + e.getMessage()); + } + return stream; + } + + protected byte[] scannerToIpcBytes(LanceScanner scanner) { + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ArrowReader reader = scanner.scanBatches()) { + try (ArrowStreamWriter writer = + new ArrowStreamWriter(reader.getVectorSchemaRoot(), null, baos)) { + writer.start(); + while (reader.loadNextBatch()) { + writer.writeBatch(); + } + } + } + return baos.toByteArray(); + } catch (IOException e) { + throw new InternalException("Failed to serialize scan results: " + e.getMessage()); + } + } + + private T withTagNotFoundHandling(String tagName, Supplier action) { + try { + return action.get(); + } catch (LanceNamespaceException e) { + throw e; + } catch (RuntimeException e) { + if (isTagNotFound(e)) { + throw new TableTagNotFoundException("Tag not found: " + tagName); + } + throw e; + } + } + + // Heuristic: the Lance SDK does not expose typed exceptions for tag operations, + // so we detect "tag not found" by inspecting the exception message. This is fragile + // and may need updating if the SDK changes its error message format. + private boolean isTagNotFound(RuntimeException e) { + String msg = e.getMessage(); + if (msg == null) { + return false; + } + String lower = msg.toLowerCase(); + return lower.contains("tag") && lower.contains("not found"); + } + + // ========== Private Helper Methods ========== + + /** + * Apply limit-based pagination to a sorted list. Uses the last element as the next page token + * when there are more results than the limit. When a pageToken is provided, skips all elements up + * to and including that token value. + * + *

Returns a new list (does not mutate the input). The next page token (or null) is stored in + * {@code nextTokenOut[0]}. + */ + private List paginateList( + List sorted, Integer limit, String pageToken, String[] nextTokenOut) { + int startIdx = 0; + + // Skip past the page token if provided + if (pageToken != null && !pageToken.isEmpty()) { + int idx = Collections.binarySearch(sorted, pageToken); + if (idx >= 0) { + startIdx = idx + 1; + } else { + startIdx = -(idx + 1); + } + } + + int endIdx = sorted.size(); + nextTokenOut[0] = null; + + if (limit != null && limit > 0 && (endIdx - startIdx) > limit) { + endIdx = startIdx + limit; + nextTokenOut[0] = sorted.get(endIdx - 1); + } + + return new ArrayList<>(sorted.subList(startIdx, endIdx)); + } + + private void closeQuietly(AutoCloseable closeable) { + try { + closeable.close(); + } catch (Exception ignored) { + // best-effort close + } + } + + private TableVersion toTableVersion(Version version) { + TableVersion tv = new TableVersion(); + tv.setVersion(version.getId()); + if (version.getDataTime() != null) { + tv.setTimestampMillis(version.getDataTime().toInstant().toEpochMilli()); + } + tv.setMetadata(version.getMetadata()); + return tv; + } + + private Map buildFieldIdToNameMap(Dataset dataset) { + Map map = new HashMap<>(); + LanceSchema lanceSchema = dataset.getLanceSchema(); + for (LanceField field : lanceSchema.fields()) { + map.put(field.getId(), field.getName()); + } + return map; + } + + private List namespacePath(List id) { + if (id.size() <= 1) { + return Collections.emptyList(); + } + return id.subList(0, id.size() - 1); + } + + private String tableName(List id) { + return id.get(id.size() - 1); + } + + private void ensureNamespaceExists(Path nsPath, List id) { + if (!Files.exists(nsPath) || !Files.isDirectory(nsPath)) { + String name = (id == null || id.isEmpty()) ? "" : String.join("/", id); + throw new NamespaceNotFoundException("Namespace not found: " + name); + } + } + + private void ensureTableExists(List tableId) { + Path tablePath = Paths.get(resolveTableUri(tableId)); + if (!Files.exists(tablePath)) { + throw new TableNotFoundException("Table not found: " + tableName(tableId)); + } + } + + private DistanceType parseDistanceType(String value) { + switch (value.toLowerCase()) { + case "l2": + case "euclidean": + return DistanceType.L2; + case "cosine": + return DistanceType.Cosine; + case "dot": + return DistanceType.Dot; + case "hamming": + return DistanceType.Hamming; + default: + throw new InvalidInputException("Unknown distance type: " + value); + } + } + + private void ensureParentNamespaceExists(List tableId) { + List nsPath = namespacePath(tableId); + if (!nsPath.isEmpty()) { + Path parentPath = resolveNamespacePath(nsPath); + if (!Files.exists(parentPath)) { + throw new NamespaceNotFoundException( + "Parent namespace not found: " + String.join("/", nsPath)); + } + } + } + + // TODO: forward fastSearch, lowerBound, upperBound once the Lance Java SDK supports them. + private ScanOptions buildScanOptions(QueryTableRequest request) { + ScanOptions.Builder builder = new ScanOptions.Builder(); + + QueryTableRequestColumns columns = request.getColumns(); + if (columns != null && columns.getColumnNames() != null) { + builder.columns(columns.getColumnNames()); + } + + String filter = request.getFilter(); + if (filter != null && !filter.isEmpty()) { + builder.filter(filter); + } + + Integer k = request.getK(); + if (k != null && k > 0) { + Query.Builder queryBuilder = new Query.Builder(); + queryBuilder.setK(k); + + QueryTableRequestVector vector = request.getVector(); + if (vector != null && vector.getSingleVector() != null) { + List values = vector.getSingleVector(); + float[] keyArray = new float[values.size()]; + for (int i = 0; i < values.size(); i++) { + keyArray[i] = values.get(i); + } + queryBuilder.setKey(keyArray); + } + + String vectorColumn = request.getVectorColumn(); + if (vectorColumn != null) { + queryBuilder.setColumn(vectorColumn); + } + + Integer nprobes = request.getNprobes(); + if (nprobes != null) { + queryBuilder.setNprobes(nprobes); + } + + Integer ef = request.getEf(); + if (ef != null) { + queryBuilder.setEf(ef); + } + + Integer refineFactor = request.getRefineFactor(); + if (refineFactor != null) { + queryBuilder.setRefineFactor(refineFactor); + } + + String distanceType = request.getDistanceType(); + if (distanceType != null) { + queryBuilder.setDistanceType(parseDistanceType(distanceType)); + } + + if (Boolean.TRUE.equals(request.getBypassVectorIndex())) { + queryBuilder.setUseIndex(false); + } + + builder.nearest(queryBuilder.build()); + } + + QueryTableRequestFullTextQuery ftsQuery = request.getFullTextQuery(); + if (ftsQuery != null && ftsQuery.getStringQuery() != null) { + StringFtsQuery stringFts = ftsQuery.getStringQuery(); + String queryText = stringFts.getQuery(); + List ftsColumns = stringFts.getColumns(); + if (queryText != null) { + if (ftsColumns != null && ftsColumns.size() == 1) { + builder.fullTextQuery(FullTextQuery.match(queryText, ftsColumns.get(0))); + } else if (ftsColumns != null && ftsColumns.size() > 1) { + builder.fullTextQuery(FullTextQuery.multiMatch(queryText, ftsColumns)); + } + } + } + + Integer offset = request.getOffset(); + if (offset != null) { + builder.offset(offset.longValue()); + } + + Boolean prefilter = request.getPrefilter(); + if (prefilter != null) { + builder.prefilter(prefilter); + } + + Boolean withRowId = request.getWithRowId(); + if (withRowId != null) { + builder.withRowId(withRowId); + } + + return builder.build(); + } + + private MergeInsertParams buildMergeInsertParams(MergeInsertIntoTableRequest request) { + String on = request.getOn(); + if (on == null || on.isEmpty()) { + throw new InvalidInputException("'on' column(s) required for merge insert"); + } + + List onColumns = new ArrayList<>(); + for (String col : on.split(",")) { + onColumns.add(col.trim()); + } + + MergeInsertParams params = new MergeInsertParams(onColumns); + + if (Boolean.TRUE.equals(request.getWhenMatchedUpdateAll())) { + String filt = request.getWhenMatchedUpdateAllFilt(); + if (filt != null && !filt.isEmpty()) { + params.withMatchedUpdateIf(filt); + } else { + params.withMatchedUpdateAll(); + } + } + + if (Boolean.TRUE.equals(request.getWhenNotMatchedInsertAll())) { + params.withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll); + } + + if (Boolean.TRUE.equals(request.getWhenNotMatchedBySourceDelete())) { + String filt = request.getWhenNotMatchedBySourceDeleteFilt(); + if (filt != null && !filt.isEmpty()) { + params.withNotMatchedBySourceDeleteIf(filt); + } else { + params.withNotMatchedBySourceDelete(); + } + } + + return params; + } + + private IndexOptions buildIndexOptions(CreateTableIndexRequest request, boolean forceScalar) { + List columns = new ArrayList<>(); + if (request.getColumn() != null) { + columns.add(request.getColumn()); + } + + String requestedType = + request.getIndexType() != null + ? request.getIndexType().toUpperCase() + : (forceScalar ? "BTREE" : "IVF_PQ"); + + IndexType indexType; + IndexParams indexParams; + switch (requestedType) { + case "BTREE": + indexType = IndexType.BTREE; + indexParams = + IndexParams.builder().setScalarIndexParams(ScalarIndexParams.create("btree")).build(); + break; + case "BITMAP": + indexType = IndexType.BITMAP; + indexParams = + IndexParams.builder().setScalarIndexParams(ScalarIndexParams.create("bitmap")).build(); + break; + case "INVERTED": + case "FTS": + indexType = IndexType.INVERTED; + indexParams = + IndexParams.builder() + .setScalarIndexParams(ScalarIndexParams.create("inverted")) + .build(); + break; + default: + if (forceScalar) { + indexType = IndexType.BTREE; + indexParams = + IndexParams.builder().setScalarIndexParams(ScalarIndexParams.create("btree")).build(); + } else { + indexType = IndexType.IVF_PQ; + indexParams = IndexParams.builder().build(); + } + break; + } + + IndexOptions.Builder builder = IndexOptions.builder(columns, indexType, indexParams); + + String indexName = request.getName(); + if (indexName != null) { + builder.withIndexName(indexName); + } + + builder.replace(true); + + return builder.build(); + } + + private JsonArrowSchema convertArrowSchemaToJson(Schema schema) { + JsonArrowSchema jsonSchema = new JsonArrowSchema(); + List fields = new ArrayList<>(); + for (org.apache.arrow.vector.types.pojo.Field field : schema.getFields()) { + fields.add(convertArrowFieldToJson(field)); + } + jsonSchema.setFields(fields); + if (schema.getCustomMetadata() != null && !schema.getCustomMetadata().isEmpty()) { + jsonSchema.setMetadata(schema.getCustomMetadata()); + } + return jsonSchema; + } + + private JsonArrowField convertArrowFieldToJson(org.apache.arrow.vector.types.pojo.Field field) { + JsonArrowField jsonField = new JsonArrowField(); + jsonField.setName(field.getName()); + jsonField.setNullable(field.isNullable()); + JsonArrowDataType dataType = new JsonArrowDataType(); + dataType.setType(field.getType().toString()); + jsonField.setType(dataType); + if (field.getMetadata() != null && !field.getMetadata().isEmpty()) { + jsonField.setMetadata(field.getMetadata()); + } + return jsonField; + } + + private void deleteRecursive(Path path) throws IOException { + if (Files.isDirectory(path)) { + try (DirectoryStream stream = Files.newDirectoryStream(path)) { + for (Path entry : stream) { + deleteRecursive(entry); + } + } + } + Files.delete(path); + } + + private void collectTablesRecursive(Path dir, String prefix, List tables) { + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + for (Path entry : stream) { + String name = entry.getFileName().toString(); + if (Files.isDirectory(entry)) { + if (name.endsWith(LANCE_EXTENSION)) { + String tableName = name.substring(0, name.length() - LANCE_EXTENSION.length()); + tables.add(prefix.isEmpty() ? tableName : prefix + "/" + tableName); + } else { + String newPrefix = prefix.isEmpty() ? name : prefix + "/" + name; + collectTablesRecursive(entry, newPrefix, tables); + } + } + } + } catch (IOException e) { + // Skip directories that can't be read + } + } +} diff --git a/java/lance-namespace-base/src/test/java/org/lance/namespace/base/BaseLanceNamespaceTest.java b/java/lance-namespace-base/src/test/java/org/lance/namespace/base/BaseLanceNamespaceTest.java new file mode 100644 index 00000000..661f9030 --- /dev/null +++ b/java/lance-namespace-base/src/test/java/org/lance/namespace/base/BaseLanceNamespaceTest.java @@ -0,0 +1,1647 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.namespace.base; + +import org.lance.namespace.errors.ErrorCode; +import org.lance.namespace.errors.InvalidInputException; +import org.lance.namespace.errors.LanceNamespaceException; +import org.lance.namespace.errors.UnsupportedOperationException; +import org.lance.namespace.model.*; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** Tests for BaseLanceNamespace implementation. */ +public class BaseLanceNamespaceTest { + @TempDir Path tempDir; + + private BufferAllocator allocator; + private BaseLanceNamespace namespace; + + @BeforeEach + void setUp() { + allocator = new RootAllocator(Long.MAX_VALUE); + namespace = new BaseLanceNamespace(); + + Map config = new HashMap<>(); + config.put("root", tempDir.toString()); + namespace.initialize(config, allocator); + } + + @AfterEach + void tearDown() { + if (namespace != null) { + namespace.close(); + } + if (allocator != null) { + allocator.close(); + } + } + + private byte[] createArrowData(int[] ids, String[] names, int[] ages) throws Exception { + Schema schema = + new Schema( + Arrays.asList( + new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field("name", FieldType.nullable(new ArrowType.Utf8()), null), + new Field("age", FieldType.nullable(new ArrowType.Int(32, true)), null))); + + try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + IntVector idVector = (IntVector) root.getVector("id"); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + IntVector ageVector = (IntVector) root.getVector("age"); + + idVector.allocateNew(ids.length); + nameVector.allocateNew(ids.length); + ageVector.allocateNew(ids.length); + + for (int i = 0; i < ids.length; i++) { + idVector.set(i, ids[i]); + nameVector.set(i, names[i].getBytes()); + ageVector.set(i, ages[i]); + } + + idVector.setValueCount(ids.length); + nameVector.setValueCount(ids.length); + ageVector.setValueCount(ids.length); + root.setRowCount(ids.length); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, out)) { + writer.writeBatch(); + } + return out.toByteArray(); + } + } + + private byte[] createTestTableData() throws Exception { + return createArrowData( + new int[] {1, 2, 3}, new String[] {"Alice", "Bob", "Charlie"}, new int[] {30, 25, 35}); + } + + private byte[] createAdditionalData() throws Exception { + return createArrowData(new int[] {4, 5}, new String[] {"Dave", "Eve"}, new int[] {40, 28}); + } + + private void createNamespace(String... parts) { + namespace.createNamespace(new CreateNamespaceRequest().id(Arrays.asList(parts))); + } + + private void createTestTable(String... idParts) throws Exception { + byte[] data = createTestTableData(); + namespace.createTable(new CreateTableRequest().id(Arrays.asList(idParts)), data); + } + + private String getTablePath(String... idParts) { + StringBuilder sb = new StringBuilder(tempDir.toString()); + for (String part : idParts) { + sb.append("/").append(part); + } + sb.append(".lance"); + return sb.toString(); + } + + private int countRowsInIpc(byte[] ipcData) throws Exception { + int totalRows = 0; + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(ipcData), allocator)) { + while (reader.loadNextBatch()) { + totalRows += reader.getVectorSchemaRoot().getRowCount(); + } + } + return totalRows; + } + + // ========== Namespace Operations ========== + + @Test + void testNamespaceId() { + String namespaceId = namespace.namespaceId(); + assertNotNull(namespaceId); + assertTrue(namespaceId.contains("BaseLanceNamespace")); + } + + @Test + void testInitializeWithoutRoot() { + BaseLanceNamespace ns = new BaseLanceNamespace(); + assertThrows(InvalidInputException.class, () -> ns.initialize(new HashMap<>(), allocator)); + } + + @Test + void testInitializeWithEmptyRoot() { + BaseLanceNamespace ns = new BaseLanceNamespace(); + Map config = new HashMap<>(); + config.put("root", ""); + assertThrows(InvalidInputException.class, () -> ns.initialize(config, allocator)); + } + + @Test + void testInitializeExtractsStorageOptions() { + BaseLanceNamespace ns = new BaseLanceNamespace(); + Map config = new HashMap<>(); + config.put("root", tempDir.toString()); + config.put("storage.region", "us-east-1"); + config.put("storage.endpoint", "http://localhost:9000"); + config.put("other.key", "value"); + ns.initialize(config, allocator); + // Namespace should be functional (storage options are used internally) + assertNotNull(ns.namespaceId()); + ns.close(); + } + + @Test + void testCreateAndListNamespaces() { + createNamespace("workspace"); + + ListNamespacesResponse listResp = namespace.listNamespaces(new ListNamespacesRequest()); + assertNotNull(listResp); + assertNotNull(listResp.getNamespaces()); + assertEquals(1, listResp.getNamespaces().size()); + assertTrue(listResp.getNamespaces().contains("workspace")); + } + + @Test + void testCreateNamespaceAlreadyExists() { + createNamespace("workspace"); + + LanceNamespaceException ex = + assertThrows(LanceNamespaceException.class, () -> createNamespace("workspace")); + assertEquals(ErrorCode.NAMESPACE_ALREADY_EXISTS, ex.getErrorCode()); + } + + @Test + void testDescribeNamespace() { + createNamespace("workspace"); + + DescribeNamespaceResponse descResp = + namespace.describeNamespace(new DescribeNamespaceRequest().id(Arrays.asList("workspace"))); + assertNotNull(descResp); + assertNotNull(descResp.getProperties()); + } + + @Test + void testNamespaceExists() { + createNamespace("workspace"); + + assertDoesNotThrow( + () -> + namespace.namespaceExists(new NamespaceExistsRequest().id(Arrays.asList("workspace")))); + + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.namespaceExists( + new NamespaceExistsRequest().id(Arrays.asList("nonexistent")))); + assertEquals(ErrorCode.NAMESPACE_NOT_FOUND, ex.getErrorCode()); + } + + @Test + void testDropNamespace() { + createNamespace("workspace"); + + DropNamespaceResponse dropResp = + namespace.dropNamespace(new DropNamespaceRequest().id(Arrays.asList("workspace"))); + assertNotNull(dropResp); + + assertThrows( + LanceNamespaceException.class, + () -> + namespace.namespaceExists(new NamespaceExistsRequest().id(Arrays.asList("workspace")))); + } + + @Test + void testDropNamespaceNotEmpty() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.dropNamespace(new DropNamespaceRequest().id(Arrays.asList("workspace")))); + assertEquals(ErrorCode.NAMESPACE_NOT_EMPTY, ex.getErrorCode()); + } + + @Test + void testListNamespacesInNestedNamespace() { + createNamespace("org"); + createNamespace("org", "team1"); + createNamespace("org", "team2"); + + ListNamespacesResponse listResp = + namespace.listNamespaces(new ListNamespacesRequest().id(Arrays.asList("org"))); + assertNotNull(listResp); + assertEquals(2, listResp.getNamespaces().size()); + assertTrue(listResp.getNamespaces().contains("team1")); + assertTrue(listResp.getNamespaces().contains("team2")); + } + + @Test + void testListNamespacesExcludesLanceTables() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "my_table"); + createNamespace("workspace", "child_ns"); + + ListNamespacesResponse listResp = + namespace.listNamespaces(new ListNamespacesRequest().id(Arrays.asList("workspace"))); + assertEquals(1, listResp.getNamespaces().size()); + assertTrue(listResp.getNamespaces().contains("child_ns")); + } + + @Test + void testDropNamespaceNotFound() { + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.dropNamespace( + new DropNamespaceRequest().id(Arrays.asList("nonexistent")))); + assertEquals(ErrorCode.NAMESPACE_NOT_FOUND, ex.getErrorCode()); + } + + @Test + void testNamespaceNotFound() { + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.describeNamespace( + new DescribeNamespaceRequest().id(Arrays.asList("nonexistent")))); + assertEquals(ErrorCode.NAMESPACE_NOT_FOUND, ex.getErrorCode()); + } + + // ========== Table Operations ========== + + @Test + void testCreateTable() throws Exception { + createNamespace("workspace"); + + byte[] tableData = createTestTableData(); + CreateTableResponse createResp = + namespace.createTable( + new CreateTableRequest().id(Arrays.asList("workspace", "test_table")), tableData); + + assertNotNull(createResp); + assertNotNull(createResp.getLocation()); + assertTrue(createResp.getLocation().contains("test_table")); + assertEquals(Long.valueOf(1), createResp.getVersion()); + } + + @Test + void testCreateTableAlreadyExists() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + byte[] tableData = createTestTableData(); + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.createTable( + new CreateTableRequest().id(Arrays.asList("workspace", "test_table")), + tableData)); + assertEquals(ErrorCode.TABLE_ALREADY_EXISTS, ex.getErrorCode()); + } + + @Test + void testListTables() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "table1"); + createTestTable("workspace", "table2"); + + ListTablesResponse listResp = + namespace.listTables(new ListTablesRequest().id(Arrays.asList("workspace"))); + + assertNotNull(listResp); + assertNotNull(listResp.getTables()); + assertEquals(2, listResp.getTables().size()); + assertTrue(listResp.getTables().contains("table1")); + assertTrue(listResp.getTables().contains("table2")); + } + + @Test + void testDescribeTable() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + DescribeTableResponse descResp = + namespace.describeTable( + new DescribeTableRequest().id(Arrays.asList("workspace", "test_table"))); + + assertNotNull(descResp); + assertNotNull(descResp.getLocation()); + assertTrue(descResp.getLocation().contains("test_table")); + assertEquals(Long.valueOf(1), descResp.getVersion()); + assertNotNull(descResp.getSchema()); + assertEquals(3, descResp.getSchema().getFields().size()); + assertEquals( + "id", descResp.getSchema().getFields().get(0).getName(), "First field should be 'id'"); + } + + @Test + void testTableExists() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + assertDoesNotThrow( + () -> + namespace.tableExists( + new TableExistsRequest().id(Arrays.asList("workspace", "test_table")))); + + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.tableExists( + new TableExistsRequest().id(Arrays.asList("workspace", "nonexistent")))); + assertEquals(ErrorCode.TABLE_NOT_FOUND, ex.getErrorCode()); + } + + @Test + void testDropTable() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + DropTableResponse dropResp = + namespace.dropTable(new DropTableRequest().id(Arrays.asList("workspace", "test_table"))); + assertNotNull(dropResp); + + assertThrows( + LanceNamespaceException.class, + () -> + namespace.tableExists( + new TableExistsRequest().id(Arrays.asList("workspace", "test_table")))); + } + + @Test + void testTableNotFound() { + createNamespace("workspace"); + + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.describeTable( + new DescribeTableRequest().id(Arrays.asList("workspace", "nonexistent")))); + assertEquals(ErrorCode.TABLE_NOT_FOUND, ex.getErrorCode()); + } + + @Test + void testCreateTableInNonexistentNamespace() throws Exception { + byte[] data = createTestTableData(); + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.createTable( + new CreateTableRequest().id(Arrays.asList("nonexistent", "table")), data)); + assertEquals(ErrorCode.NAMESPACE_NOT_FOUND, ex.getErrorCode()); + } + + @Test + void testDeclareTableAlreadyExists() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.declareTable( + new DeclareTableRequest().id(Arrays.asList("workspace", "test_table")))); + assertEquals(ErrorCode.TABLE_ALREADY_EXISTS, ex.getErrorCode()); + } + + @Test + void testCountTableRows() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + long count = + namespace.countTableRows( + new CountTableRowsRequest().id(Arrays.asList("workspace", "test_table"))); + assertEquals(3, count); + } + + @Test + void testCountTableRowsWithPredicate() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + long count = + namespace.countTableRows( + new CountTableRowsRequest() + .id(Arrays.asList("workspace", "test_table")) + .predicate("age > 28")); + assertEquals(2, count); // Alice (30) and Charlie (35) + } + + @Test + void testDeclareTable() { + createNamespace("workspace"); + + DeclareTableResponse declResp = + namespace.declareTable( + new DeclareTableRequest().id(Arrays.asList("workspace", "declared_table"))); + assertNotNull(declResp); + assertNotNull(declResp.getLocation()); + assertTrue(declResp.getLocation().contains("declared_table")); + + // Verify declared table is visible in table listing + ListTablesResponse listResp = + namespace.listTables(new ListTablesRequest().id(Arrays.asList("workspace"))); + assertTrue(listResp.getTables().contains("declared_table")); + } + + // ========== Data Operations ========== + + @Test + void testInsertIntoTable() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + byte[] additionalData = createAdditionalData(); + InsertIntoTableResponse insertResp = + namespace.insertIntoTable( + new InsertIntoTableRequest().id(Arrays.asList("workspace", "test_table")), + additionalData); + assertNotNull(insertResp); + + long count = + namespace.countTableRows( + new CountTableRowsRequest().id(Arrays.asList("workspace", "test_table"))); + assertEquals(5, count); + } + + @Test + void testQueryTable() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + byte[] result = + namespace.queryTable(new QueryTableRequest().id(Arrays.asList("workspace", "test_table"))); + assertNotNull(result); + assertTrue(result.length > 0); + + int rowCount = countRowsInIpc(result); + assertEquals(3, rowCount); + } + + @Test + void testQueryTableWithFilter() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + QueryTableRequest queryReq = + new QueryTableRequest().id(Arrays.asList("workspace", "test_table")).filter("age > 28"); + byte[] result = namespace.queryTable(queryReq); + assertNotNull(result); + + int rowCount = countRowsInIpc(result); + assertEquals(2, rowCount); // Alice (30) and Charlie (35) + } + + @Test + void testQueryTableWithProjection() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + QueryTableRequestColumns columns = new QueryTableRequestColumns(); + columns.setColumnNames(Arrays.asList("id", "name")); + QueryTableRequest queryReq = + new QueryTableRequest().id(Arrays.asList("workspace", "test_table")).columns(columns); + byte[] result = namespace.queryTable(queryReq); + assertNotNull(result); + + // Verify schema only has 2 columns + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(result), allocator)) { + Schema schema = reader.getVectorSchemaRoot().getSchema(); + assertEquals(2, schema.getFields().size()); + assertEquals("id", schema.getFields().get(0).getName()); + assertEquals("name", schema.getFields().get(1).getName()); + } + } + + @Test + void testDeleteFromTable() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + namespace.deleteFromTable( + new DeleteFromTableRequest() + .id(Arrays.asList("workspace", "test_table")) + .predicate("id = 2")); + + long count = + namespace.countTableRows( + new CountTableRowsRequest().id(Arrays.asList("workspace", "test_table"))); + assertEquals(2, count); + } + + @Test + void testDeleteFromTableWithoutPredicate() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + assertThrows( + InvalidInputException.class, + () -> + namespace.deleteFromTable( + new DeleteFromTableRequest().id(Arrays.asList("workspace", "test_table")))); + } + + @Test + void testDeleteFromTableWithEmptyPredicate() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + assertThrows( + InvalidInputException.class, + () -> + namespace.deleteFromTable( + new DeleteFromTableRequest() + .id(Arrays.asList("workspace", "test_table")) + .predicate(""))); + } + + @Test + void testQueryTableWithVersion() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Insert more data (creates version 2) + byte[] additionalData = createAdditionalData(); + namespace.insertIntoTable(new InsertIntoTableRequest().id(tableId), additionalData); + + // Query at version 1 should return only 3 rows + byte[] result = namespace.queryTable(new QueryTableRequest().id(tableId).version(1L)); + assertEquals(3, countRowsInIpc(result)); + + // Query at latest (no version) should return 5 rows + result = namespace.queryTable(new QueryTableRequest().id(tableId)); + assertEquals(5, countRowsInIpc(result)); + } + + @Test + void testMergeInsertIntoTable() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + // Create merge data: update existing id=1, insert new id=4 + byte[] mergeData = + createArrowData( + new int[] {1, 4}, new String[] {"Alice Updated", "Dave"}, new int[] {31, 40}); + + MergeInsertIntoTableRequest mergeReq = new MergeInsertIntoTableRequest(); + mergeReq.setId(Arrays.asList("workspace", "test_table")); + mergeReq.setOn("id"); + mergeReq.setWhenMatchedUpdateAll(true); + mergeReq.setWhenNotMatchedInsertAll(true); + + MergeInsertIntoTableResponse mergeResp = namespace.mergeInsertIntoTable(mergeReq, mergeData); + assertNotNull(mergeResp); + + long count = + namespace.countTableRows( + new CountTableRowsRequest().id(Arrays.asList("workspace", "test_table"))); + assertEquals( + 4, count); // 3 original - 1 updated + 1 new = 4 (Alice updated, Bob, Charlie, Dave) + } + + @Test + void testMergeInsertWithDeleteNotMatchedBySource() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Merge: keep only id=1, delete anything not in source + byte[] mergeData = createArrowData(new int[] {1}, new String[] {"Alice"}, new int[] {30}); + + MergeInsertIntoTableRequest mergeReq = new MergeInsertIntoTableRequest(); + mergeReq.setId(tableId); + mergeReq.setOn("id"); + mergeReq.setWhenMatchedUpdateAll(true); + mergeReq.setWhenNotMatchedBySourceDelete(true); + + MergeInsertIntoTableResponse mergeResp = namespace.mergeInsertIntoTable(mergeReq, mergeData); + assertNotNull(mergeResp); + + long count = namespace.countTableRows(new CountTableRowsRequest().id(tableId)); + assertEquals(1, count); // Only Alice remains + } + + @Test + void testMergeInsertWithoutOnColumn() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + byte[] mergeData = createArrowData(new int[] {1}, new String[] {"Alice"}, new int[] {30}); + + MergeInsertIntoTableRequest mergeReq = new MergeInsertIntoTableRequest(); + mergeReq.setId(Arrays.asList("workspace", "test_table")); + + assertThrows( + InvalidInputException.class, () -> namespace.mergeInsertIntoTable(mergeReq, mergeData)); + } + + // ========== Index Operations ========== + + @Test + void testCreateAndListScalarIndex() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Create scalar index on 'id' column + CreateTableIndexRequest indexReq = new CreateTableIndexRequest(); + indexReq.setId(tableId); + indexReq.setColumn("id"); + indexReq.setIndexType("btree"); + + CreateTableScalarIndexResponse indexResp = namespace.createTableScalarIndex(indexReq); + assertNotNull(indexResp); + + // List indices + ListTableIndicesResponse listResp = + namespace.listTableIndices(new ListTableIndicesRequest().id(tableId)); + assertNotNull(listResp); + assertNotNull(listResp.getIndexes()); + assertEquals(1, listResp.getIndexes().size()); + assertEquals( + Arrays.asList("id"), + listResp.getIndexes().get(0).getColumns(), + "Index should reference the 'id' column by name"); + } + + @Test + void testDropTableIndex() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Create index + CreateTableIndexRequest indexReq = new CreateTableIndexRequest(); + indexReq.setId(tableId); + indexReq.setColumn("id"); + indexReq.setIndexType("btree"); + namespace.createTableScalarIndex(indexReq); + + // List and find index name + ListTableIndicesResponse listResp = + namespace.listTableIndices(new ListTableIndicesRequest().id(tableId)); + String indexName = listResp.getIndexes().get(0).getIndexName(); + + // Drop it + DropTableIndexResponse dropResp = + namespace.dropTableIndex(new DropTableIndexRequest().id(tableId), indexName); + assertNotNull(dropResp); + + // Verify index is gone + ListTableIndicesResponse afterDrop = + namespace.listTableIndices(new ListTableIndicesRequest().id(tableId)); + assertEquals(0, afterDrop.getIndexes().size()); + } + + @Test + void testDescribeTableIndexStats() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Create index + CreateTableIndexRequest indexReq = new CreateTableIndexRequest(); + indexReq.setId(tableId); + indexReq.setColumn("id"); + indexReq.setIndexType("btree"); + namespace.createTableScalarIndex(indexReq); + + // Get index name + ListTableIndicesResponse listResp = + namespace.listTableIndices(new ListTableIndicesRequest().id(tableId)); + String indexName = listResp.getIndexes().get(0).getIndexName(); + + // Describe stats + DescribeTableIndexStatsResponse statsResp = + namespace.describeTableIndexStats( + new DescribeTableIndexStatsRequest().id(tableId), indexName); + assertNotNull(statsResp); + assertNotNull(statsResp.getIndexType()); + assertNotNull(statsResp.getNumIndexedRows()); + assertNotNull(statsResp.getNumIndices()); + } + + @Test + void testDescribeTableIndexStatsNotFound() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.describeTableIndexStats( + new DescribeTableIndexStatsRequest().id(tableId), "nonexistent_index")); + assertEquals(ErrorCode.TABLE_INDEX_NOT_FOUND, ex.getErrorCode()); + } + + // ========== Schema Operations ========== + + @Test + void testAlterTableAddColumns() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + NewColumnTransform newCol = new NewColumnTransform(); + newCol.setName("score"); + newCol.setExpression("0"); + + AlterTableAddColumnsRequest addReq = new AlterTableAddColumnsRequest(); + addReq.setId(tableId); + addReq.setNewColumns(Arrays.asList(newCol)); + + AlterTableAddColumnsResponse addResp = namespace.alterTableAddColumns(addReq); + assertNotNull(addResp); + + // Verify the column exists via describe + DescribeTableResponse descResp = + namespace.describeTable(new DescribeTableRequest().id(tableId)); + assertNotNull(descResp.getSchema()); + boolean hasScoreField = + descResp.getSchema().getFields().stream().anyMatch(f -> "score".equals(f.getName())); + assertTrue(hasScoreField, "Schema should contain 'score' column after add"); + assertEquals( + 4, descResp.getSchema().getFields().size(), "Schema should have 4 fields after add"); + } + + @Test + void testAlterTableDropColumns() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + AlterTableDropColumnsRequest dropReq = new AlterTableDropColumnsRequest(); + dropReq.setId(tableId); + dropReq.setColumns(Arrays.asList("age")); + + AlterTableDropColumnsResponse dropResp = namespace.alterTableDropColumns(dropReq); + assertNotNull(dropResp); + + // Verify column removed + DescribeTableResponse descResp = + namespace.describeTable(new DescribeTableRequest().id(tableId)); + boolean hasAgeField = + descResp.getSchema().getFields().stream().anyMatch(f -> "age".equals(f.getName())); + assertFalse(hasAgeField, "Schema should not contain 'age' column after drop"); + } + + @Test + void testAlterTableAlterColumns() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + AlterColumnsEntry entry = new AlterColumnsEntry(); + entry.setPath("name"); + entry.setRename("full_name"); + + AlterTableAlterColumnsRequest alterReq = new AlterTableAlterColumnsRequest(); + alterReq.setId(tableId); + alterReq.setAlterations(Arrays.asList(entry)); + + AlterTableAlterColumnsResponse alterResp = namespace.alterTableAlterColumns(alterReq); + assertNotNull(alterResp); + + // Verify rename + DescribeTableResponse descResp = + namespace.describeTable(new DescribeTableRequest().id(tableId)); + boolean hasFullName = + descResp.getSchema().getFields().stream().anyMatch(f -> "full_name".equals(f.getName())); + assertTrue(hasFullName, "Schema should contain 'full_name' after rename"); + boolean hasName = + descResp.getSchema().getFields().stream().anyMatch(f -> "name".equals(f.getName())); + assertFalse(hasName, "Schema should not contain 'name' after rename"); + } + + @Test + void testAlterTableAddColumnsEmpty() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + AlterTableAddColumnsRequest addReq = new AlterTableAddColumnsRequest(); + addReq.setId(Arrays.asList("workspace", "test_table")); + addReq.setNewColumns(Collections.emptyList()); + + assertThrows(InvalidInputException.class, () -> namespace.alterTableAddColumns(addReq)); + } + + @Test + void testAlterTableDropColumnsEmpty() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + AlterTableDropColumnsRequest dropReq = new AlterTableDropColumnsRequest(); + dropReq.setId(Arrays.asList("workspace", "test_table")); + dropReq.setColumns(Collections.emptyList()); + + assertThrows(InvalidInputException.class, () -> namespace.alterTableDropColumns(dropReq)); + } + + @Test + void testAlterTableAlterColumnsEmpty() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + AlterTableAlterColumnsRequest alterReq = new AlterTableAlterColumnsRequest(); + alterReq.setId(Arrays.asList("workspace", "test_table")); + alterReq.setAlterations(Collections.emptyList()); + + assertThrows(InvalidInputException.class, () -> namespace.alterTableAlterColumns(alterReq)); + } + + @Test + void testGetTableStats() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + GetTableStatsResponse statsResp = + namespace.getTableStats( + new GetTableStatsRequest().id(Arrays.asList("workspace", "test_table"))); + assertNotNull(statsResp); + assertEquals(Long.valueOf(3), statsResp.getNumRows()); + } + + // ========== Version Operations ========== + + @Test + void testListTableVersions() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Insert more data to create version 2 + byte[] additionalData = createAdditionalData(); + namespace.insertIntoTable(new InsertIntoTableRequest().id(tableId), additionalData); + + ListTableVersionsResponse versionsResp = + namespace.listTableVersions(new ListTableVersionsRequest().id(tableId)); + assertNotNull(versionsResp); + assertNotNull(versionsResp.getVersions()); + assertEquals(2, versionsResp.getVersions().size()); + } + + @Test + void testRestoreTable() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Insert more data (version 2) + byte[] additionalData = createAdditionalData(); + namespace.insertIntoTable(new InsertIntoTableRequest().id(tableId), additionalData); + + // Verify 5 rows at version 2 + long count = namespace.countTableRows(new CountTableRowsRequest().id(tableId)); + assertEquals(5, count); + + // Restore to version 1 + RestoreTableResponse restoreResp = + namespace.restoreTable(new RestoreTableRequest().id(tableId).version(1L)); + assertNotNull(restoreResp); + + // Verify 3 rows after restore + count = namespace.countTableRows(new CountTableRowsRequest().id(tableId)); + assertEquals(3, count); + } + + @Test + void testDescribeTableVersion() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + DescribeTableVersionResponse resp = + namespace.describeTableVersion(new DescribeTableVersionRequest().id(tableId).version(1L)); + assertNotNull(resp); + assertNotNull(resp.getVersion()); + assertEquals(Long.valueOf(1), resp.getVersion().getVersion()); + } + + @Test + void testDescribeTableVersionNotFound() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> + namespace.describeTableVersion( + new DescribeTableVersionRequest().id(tableId).version(999L))); + assertEquals(ErrorCode.TABLE_VERSION_NOT_FOUND, ex.getErrorCode()); + } + + @Test + void testRestoreTableWithoutVersion() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + assertThrows( + InvalidInputException.class, + () -> + namespace.restoreTable( + new RestoreTableRequest().id(Arrays.asList("workspace", "test_table")))); + } + + @Test + void testRenameTable() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + RenameTableRequest renameReq = new RenameTableRequest(); + renameReq.setId(Arrays.asList("workspace", "test_table")); + renameReq.setNewTableName("renamed_table"); + + RenameTableResponse renameResp = namespace.renameTable(renameReq); + assertNotNull(renameResp); + + // Old table should not exist + assertThrows( + LanceNamespaceException.class, + () -> + namespace.tableExists( + new TableExistsRequest().id(Arrays.asList("workspace", "test_table")))); + + // New table should exist + assertDoesNotThrow( + () -> + namespace.tableExists( + new TableExistsRequest().id(Arrays.asList("workspace", "renamed_table")))); + } + + @Test + void testRenameTableToExistingName() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "table_a"); + createTestTable("workspace", "table_b"); + + RenameTableRequest renameReq = new RenameTableRequest(); + renameReq.setId(Arrays.asList("workspace", "table_a")); + renameReq.setNewTableName("table_b"); + + LanceNamespaceException ex = + assertThrows(LanceNamespaceException.class, () -> namespace.renameTable(renameReq)); + assertEquals(ErrorCode.TABLE_ALREADY_EXISTS, ex.getErrorCode()); + } + + @Test + void testRenameTableWithoutNewName() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + RenameTableRequest renameReq = new RenameTableRequest(); + renameReq.setId(Arrays.asList("workspace", "test_table")); + + assertThrows(InvalidInputException.class, () -> namespace.renameTable(renameReq)); + } + + @Test + void testListAllTables() throws Exception { + createNamespace("ns1"); + createNamespace("ns2"); + createTestTable("ns1", "table_a"); + createTestTable("ns2", "table_b"); + + ListTablesResponse allTablesResp = namespace.listAllTables(new ListTablesRequest()); + assertNotNull(allTablesResp); + assertNotNull(allTablesResp.getTables()); + assertEquals(2, allTablesResp.getTables().size()); + // Verify qualified path format: "namespace/table" + assertTrue(allTablesResp.getTables().contains("ns1/table_a")); + assertTrue(allTablesResp.getTables().contains("ns2/table_b")); + } + + @Test + void testListAllTablesWithNestedNamespaces() throws Exception { + createNamespace("org"); + createNamespace("org", "team"); + createTestTable("org", "team", "deep_table"); + createTestTable("root_table"); + + ListTablesResponse allTablesResp = namespace.listAllTables(new ListTablesRequest()); + assertNotNull(allTablesResp); + assertEquals(2, allTablesResp.getTables().size()); + assertTrue(allTablesResp.getTables().contains("org/team/deep_table")); + assertTrue(allTablesResp.getTables().contains("root_table")); + } + + // ========== Tag Operations ========== + + @Test + void testCreateAndListTags() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Create a tag + CreateTableTagRequest tagReq = new CreateTableTagRequest(); + tagReq.setId(tableId); + tagReq.setTag("v1.0"); + tagReq.setVersion(1L); + + CreateTableTagResponse tagResp = namespace.createTableTag(tagReq); + assertNotNull(tagResp); + + // List tags + ListTableTagsResponse listResp = + namespace.listTableTags(new ListTableTagsRequest().id(tableId)); + assertNotNull(listResp); + assertNotNull(listResp.getTags()); + assertTrue(listResp.getTags().containsKey("v1.0")); + assertEquals(Long.valueOf(1), listResp.getTags().get("v1.0").getVersion()); + } + + @Test + void testGetTableTagVersion() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Create a tag + CreateTableTagRequest tagReq = new CreateTableTagRequest(); + tagReq.setId(tableId); + tagReq.setTag("v1.0"); + tagReq.setVersion(1L); + namespace.createTableTag(tagReq); + + // Get tag version + GetTableTagVersionRequest getReq = new GetTableTagVersionRequest(); + getReq.setId(tableId); + getReq.setTag("v1.0"); + + GetTableTagVersionResponse getResp = namespace.getTableTagVersion(getReq); + assertNotNull(getResp); + assertEquals(Long.valueOf(1), getResp.getVersion()); + } + + @Test + void testDeleteTableTag() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Create and delete tag + CreateTableTagRequest tagReq = new CreateTableTagRequest(); + tagReq.setId(tableId); + tagReq.setTag("v1.0"); + tagReq.setVersion(1L); + namespace.createTableTag(tagReq); + + DeleteTableTagRequest deleteReq = new DeleteTableTagRequest(); + deleteReq.setId(tableId); + deleteReq.setTag("v1.0"); + + DeleteTableTagResponse deleteResp = namespace.deleteTableTag(deleteReq); + assertNotNull(deleteResp); + + // Verify tag is gone + LanceNamespaceException ex = + assertThrows( + LanceNamespaceException.class, + () -> { + GetTableTagVersionRequest getReq = new GetTableTagVersionRequest(); + getReq.setId(tableId); + getReq.setTag("v1.0"); + namespace.getTableTagVersion(getReq); + }); + assertEquals(ErrorCode.TABLE_TAG_NOT_FOUND, ex.getErrorCode()); + } + + @Test + void testCreateTableTagMissingFields() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + CreateTableTagRequest tagReq = new CreateTableTagRequest(); + tagReq.setId(Arrays.asList("workspace", "test_table")); + // Missing tag name and version + assertThrows(InvalidInputException.class, () -> namespace.createTableTag(tagReq)); + } + + @Test + void testGetTableTagVersionNotFound() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + GetTableTagVersionRequest getReq = new GetTableTagVersionRequest(); + getReq.setId(Arrays.asList("workspace", "test_table")); + getReq.setTag("nonexistent"); + + LanceNamespaceException ex = + assertThrows(LanceNamespaceException.class, () -> namespace.getTableTagVersion(getReq)); + assertEquals(ErrorCode.TABLE_TAG_NOT_FOUND, ex.getErrorCode()); + } + + @Test + void testUpdateTableTag() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Insert data to create version 2 + byte[] additionalData = createAdditionalData(); + namespace.insertIntoTable(new InsertIntoTableRequest().id(tableId), additionalData); + + // Create tag at version 1 + CreateTableTagRequest tagReq = new CreateTableTagRequest(); + tagReq.setId(tableId); + tagReq.setTag("latest"); + tagReq.setVersion(1L); + namespace.createTableTag(tagReq); + + // Update tag to version 2 + UpdateTableTagRequest updateReq = new UpdateTableTagRequest(); + updateReq.setId(tableId); + updateReq.setTag("latest"); + updateReq.setVersion(2L); + + UpdateTableTagResponse updateResp = namespace.updateTableTag(updateReq); + assertNotNull(updateResp); + + // Verify tag now points to version 2 + GetTableTagVersionRequest getReq = new GetTableTagVersionRequest(); + getReq.setId(tableId); + getReq.setTag("latest"); + GetTableTagVersionResponse getResp = namespace.getTableTagVersion(getReq); + assertEquals(Long.valueOf(2), getResp.getVersion()); + } + + // ========== Nested Namespace Tests ========== + + @Test + void testNestedNamespaces() throws Exception { + createNamespace("org"); + createNamespace("org", "team"); + createTestTable("org", "team", "dataset"); + + // Verify table accessible + assertDoesNotThrow( + () -> + namespace.tableExists( + new TableExistsRequest().id(Arrays.asList("org", "team", "dataset")))); + + long count = + namespace.countTableRows( + new CountTableRowsRequest().id(Arrays.asList("org", "team", "dataset"))); + assertEquals(3, count); + } + + // ========== Table at Root Level ========== + + @Test + void testTableAtRootLevel() throws Exception { + byte[] data = createTestTableData(); + CreateTableResponse createResp = + namespace.createTable(new CreateTableRequest().id(Arrays.asList("root_table")), data); + assertNotNull(createResp); + + long count = + namespace.countTableRows(new CountTableRowsRequest().id(Arrays.asList("root_table"))); + assertEquals(3, count); + } + + // ========== Deregister Table ========== + + @Test + void testDeregisterTable() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + DeregisterTableResponse deregResp = + namespace.deregisterTable( + new DeregisterTableRequest().id(Arrays.asList("workspace", "test_table"))); + assertNotNull(deregResp); + + // Table should be gone + assertThrows( + LanceNamespaceException.class, + () -> + namespace.tableExists( + new TableExistsRequest().id(Arrays.asList("workspace", "test_table")))); + } + + // ========== Register Table Tests ========== + + @Test + void testRegisterTable() throws Exception { + // Create a table at one location + createTestTable("source_table"); + String sourceLocation = getTablePath("source_table"); + + // Register it under a namespace + createNamespace("workspace"); + RegisterTableRequest regReq = new RegisterTableRequest(); + regReq.setId(Arrays.asList("workspace", "registered_table")); + regReq.setLocation(sourceLocation); + + RegisterTableResponse regResp = namespace.registerTable(regReq); + assertNotNull(regResp); + assertNotNull(regResp.getLocation()); + } + + @Test + void testRegisterTableWithoutLocation() { + createNamespace("workspace"); + + RegisterTableRequest regReq = new RegisterTableRequest(); + regReq.setId(Arrays.asList("workspace", "table")); + + assertThrows(InvalidInputException.class, () -> namespace.registerTable(regReq)); + } + + @Test + void testRegisterTableAlreadyExists() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "existing_table"); + createTestTable("source_table"); + String sourceLocation = getTablePath("source_table"); + + RegisterTableRequest regReq = new RegisterTableRequest(); + regReq.setId(Arrays.asList("workspace", "existing_table")); + regReq.setLocation(sourceLocation); + + LanceNamespaceException ex = + assertThrows(LanceNamespaceException.class, () -> namespace.registerTable(regReq)); + assertEquals(ErrorCode.TABLE_ALREADY_EXISTS, ex.getErrorCode()); + } + + // ========== Insert and Query Roundtrip ========== + + @Test + void testInsertQueryDeleteRoundtrip() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + + List tableId = Arrays.asList("workspace", "test_table"); + + // Verify initial data + byte[] result = namespace.queryTable(new QueryTableRequest().id(tableId)); + assertEquals(3, countRowsInIpc(result)); + + // Insert more data + byte[] additionalData = createAdditionalData(); + namespace.insertIntoTable(new InsertIntoTableRequest().id(tableId), additionalData); + + result = namespace.queryTable(new QueryTableRequest().id(tableId)); + assertEquals(5, countRowsInIpc(result)); + + // Delete some rows + namespace.deleteFromTable(new DeleteFromTableRequest().id(tableId).predicate("id >= 4")); + + result = namespace.queryTable(new QueryTableRequest().id(tableId)); + assertEquals(3, countRowsInIpc(result)); + } + + // ========== Unimplemented Methods ========== + + @Test + void testUpdateTableNotSupported() { + assertThrows( + UnsupportedOperationException.class, () -> namespace.updateTable(new UpdateTableRequest())); + } + + @Test + void testCreateTableVersionNotSupported() { + assertThrows( + UnsupportedOperationException.class, + () -> namespace.createTableVersion(new CreateTableVersionRequest())); + } + + @Test + void testBatchDeleteTableVersionsNotSupported() { + assertThrows( + UnsupportedOperationException.class, + () -> namespace.batchDeleteTableVersions(new BatchDeleteTableVersionsRequest())); + } + + @Test + void testBatchCreateTableVersionsNotSupported() { + assertThrows( + UnsupportedOperationException.class, + () -> namespace.batchCreateTableVersions(new BatchCreateTableVersionsRequest())); + } + + @Test + void testBatchCommitTablesNotSupported() { + assertThrows( + UnsupportedOperationException.class, + () -> namespace.batchCommitTables(new BatchCommitTablesRequest())); + } + + @Test + void testUpdateTableSchemaMetadataNotSupported() { + assertThrows( + UnsupportedOperationException.class, + () -> namespace.updateTableSchemaMetadata(new UpdateTableSchemaMetadataRequest())); + } + + @Test + void testExplainTableQueryPlanNotSupported() { + assertThrows( + UnsupportedOperationException.class, + () -> namespace.explainTableQueryPlan(new ExplainTableQueryPlanRequest())); + } + + @Test + void testAnalyzeTableQueryPlanNotSupported() { + assertThrows( + UnsupportedOperationException.class, + () -> namespace.analyzeTableQueryPlan(new AnalyzeTableQueryPlanRequest())); + } + + @Test + void testDescribeTransactionNotSupported() { + assertThrows( + UnsupportedOperationException.class, + () -> namespace.describeTransaction(new DescribeTransactionRequest())); + } + + @Test + void testAlterTransactionNotSupported() { + assertThrows( + UnsupportedOperationException.class, + () -> namespace.alterTransaction(new AlterTransactionRequest())); + } + + // ========== Pagination Tests ========== + + @Test + void testListNamespacesWithLimit() { + createNamespace("alpha"); + createNamespace("beta"); + createNamespace("gamma"); + + ListNamespacesRequest request = new ListNamespacesRequest(); + request.setLimit(2); + + ListNamespacesResponse response = namespace.listNamespaces(request); + assertEquals(2, response.getNamespaces().size()); + assertNotNull(response.getPageToken()); + assertTrue(response.getNamespaces().contains("alpha")); + assertTrue(response.getNamespaces().contains("beta")); + } + + @Test + void testListNamespacesWithPageToken() { + createNamespace("alpha"); + createNamespace("beta"); + createNamespace("gamma"); + + ListNamespacesRequest request = new ListNamespacesRequest(); + request.setLimit(2); + ListNamespacesResponse firstPage = namespace.listNamespaces(request); + + // Use pageToken to get next page + ListNamespacesRequest nextRequest = new ListNamespacesRequest(); + nextRequest.setPageToken(firstPage.getPageToken()); + ListNamespacesResponse secondPage = namespace.listNamespaces(nextRequest); + assertEquals(1, secondPage.getNamespaces().size()); + assertTrue(secondPage.getNamespaces().contains("gamma")); + } + + @Test + void testListTablesWithLimit() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "alpha_table"); + createTestTable("workspace", "beta_table"); + createTestTable("workspace", "gamma_table"); + + ListTablesRequest request = new ListTablesRequest(); + request.setId(Collections.singletonList("workspace")); + request.setLimit(2); + + ListTablesResponse response = namespace.listTables(request); + assertEquals(2, response.getTables().size()); + assertNotNull(response.getPageToken()); + } + + @Test + void testListTablesWithPageToken() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "alpha_table"); + createTestTable("workspace", "beta_table"); + createTestTable("workspace", "gamma_table"); + + ListTablesRequest request = new ListTablesRequest(); + request.setId(Collections.singletonList("workspace")); + request.setLimit(2); + ListTablesResponse firstPage = namespace.listTables(request); + assertNotNull(firstPage.getPageToken()); + + ListTablesRequest nextRequest = new ListTablesRequest(); + nextRequest.setId(Collections.singletonList("workspace")); + nextRequest.setPageToken(firstPage.getPageToken()); + ListTablesResponse secondPage = namespace.listTables(nextRequest); + assertEquals(1, secondPage.getTables().size()); + assertTrue(secondPage.getTables().contains("gamma_table")); + } + + @Test + void testListTableVersionsWithLimit() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + List tableId = Arrays.asList("workspace", "test_table"); + + // Insert to create additional versions + byte[] data = createAdditionalData(); + namespace.insertIntoTable(new InsertIntoTableRequest().id(tableId), data); + namespace.insertIntoTable(new InsertIntoTableRequest().id(tableId), data); + + ListTableVersionsRequest request = new ListTableVersionsRequest(); + request.setId(tableId); + // Should have at least 3 versions (create + 2 inserts); limit to 2 + request.setLimit(2); + + ListTableVersionsResponse response = namespace.listTableVersions(request); + assertEquals(2, response.getVersions().size()); + } + + @Test + void testListTableTagsWithLimit() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + List tableId = Arrays.asList("workspace", "test_table"); + + namespace.createTableTag(new CreateTableTagRequest().id(tableId).tag("v1").version(1L)); + + // Insert to create version 2 + byte[] data = createAdditionalData(); + namespace.insertIntoTable(new InsertIntoTableRequest().id(tableId), data); + namespace.createTableTag(new CreateTableTagRequest().id(tableId).tag("v2").version(2L)); + + ListTableTagsRequest request = new ListTableTagsRequest(); + request.setId(tableId); + request.setLimit(1); + + ListTableTagsResponse response = namespace.listTableTags(request); + assertEquals(1, response.getTags().size()); + } + + @Test + void testListAllTablesWithLimit() throws Exception { + createNamespace("ns_a"); + createNamespace("ns_b"); + createTestTable("ns_a", "table1"); + createTestTable("ns_a", "table2"); + createTestTable("ns_b", "table3"); + + ListTablesRequest request = new ListTablesRequest(); + request.setLimit(2); + + ListTablesResponse response = namespace.listAllTables(request); + assertEquals(2, response.getTables().size()); + assertNotNull(response.getPageToken()); + } + + @Test + void testListAllTablesWithPageToken() throws Exception { + createNamespace("ns_a"); + createNamespace("ns_b"); + createTestTable("ns_a", "table1"); + createTestTable("ns_a", "table2"); + createTestTable("ns_b", "table3"); + + ListTablesRequest request = new ListTablesRequest(); + request.setLimit(2); + ListTablesResponse firstPage = namespace.listAllTables(request); + assertNotNull(firstPage.getPageToken()); + + ListTablesRequest nextRequest = new ListTablesRequest(); + nextRequest.setPageToken(firstPage.getPageToken()); + ListTablesResponse secondPage = namespace.listAllTables(nextRequest); + assertEquals(1, secondPage.getTables().size()); + } + + @Test + void testPaginationWithNoLimit() throws Exception { + createNamespace("alpha"); + createNamespace("beta"); + createNamespace("gamma"); + + // No limit -- should return all results, no page token + ListNamespacesResponse response = namespace.listNamespaces(new ListNamespacesRequest()); + assertEquals(3, response.getNamespaces().size()); + assertNull(response.getPageToken()); + } + + // ========== Query Parameter Tests ========== + + @Test + void testQueryTableWithDistanceType() throws Exception { + // distanceType is only meaningful with vector search (k > 0), but we verify + // it doesn't throw when passed without a vector query (it's inside the k > 0 block) + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + List tableId = Arrays.asList("workspace", "test_table"); + + QueryTableRequest request = new QueryTableRequest(); + request.setId(tableId); + request.setDistanceType("cosine"); + // No k set -- distanceType is ignored gracefully + byte[] result = namespace.queryTable(request); + assertEquals(3, countRowsInIpc(result)); + } + + @Test + void testParseDistanceTypeInvalid() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + List tableId = Arrays.asList("workspace", "test_table"); + + QueryTableRequest request = new QueryTableRequest(); + request.setId(tableId); + request.setK(5); + request.setDistanceType("invalid_type"); + + // Vector with single vector is needed for k > 0 + QueryTableRequestVector vector = new QueryTableRequestVector(); + vector.setSingleVector(Arrays.asList(1.0f, 2.0f, 3.0f)); + request.setVector(vector); + + LanceNamespaceException ex = + assertThrows(LanceNamespaceException.class, () -> namespace.queryTable(request)); + assertEquals(ErrorCode.INVALID_INPUT, ex.getErrorCode()); + } + + @Test + void testQueryTableWithBypassVectorIndex() throws Exception { + createNamespace("workspace"); + createTestTable("workspace", "test_table"); + List tableId = Arrays.asList("workspace", "test_table"); + + // bypassVectorIndex without vector search -- should be ignored gracefully + QueryTableRequest request = new QueryTableRequest(); + request.setId(tableId); + request.setBypassVectorIndex(true); + + byte[] result = namespace.queryTable(request); + assertEquals(3, countRowsInIpc(result)); + } +} diff --git a/java/pom.xml b/java/pom.xml index 21eeb40c..59953ff7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -92,6 +92,7 @@ lance-namespace-springboot-server lance-namespace-core lance-namespace-core-async + lance-namespace-base