diff --git a/docs/src/impls/polaris.md b/docs/src/impls/polaris.md new file mode 100644 index 000000000..4672bfc61 --- /dev/null +++ b/docs/src/impls/polaris.md @@ -0,0 +1,54 @@ +# Lance Polaris Namespace + +**Lance Polaris Namespace** is an implementation using Polaris Catalog's Generic Table API. +The Polaris namespace uses the following API endpoints: +- **Namespace operations**: Standard Iceberg REST API endpoints (`/namespaces`) +- **Table operations**: Generic Table API endpoints (`/namespaces/{namespace}/generic-tables`) + +For more details about Polaris Catalog, please read the [Polaris Catalog Documentation](https://github.com/polaris-catalog/polaris). + +## Configuration + +The Lance Polaris namespace accepts the following configuration properties: + +| Property | Required | Description | Default | Example | +|-------------------|----------|------------------------------------------------|---------|----------------------------| +| `endpoint` | Yes | Polaris server endpoint URL | | `http://localhost:8182` | +| `auth_token` | No | Bearer token for authentication | | `your-auth-token` | +| `connect_timeout` | No | Connection timeout in milliseconds | 10000 | `30000` | +| `read_timeout` | No | Read timeout in milliseconds | 30000 | `60000` | +| `max_retries` | No | Maximum number of retries for failed requests | 3 | `5` | + +### Authentication + +The Polaris namespace supports bearer token authentication: + +1. **Bearer Token**: Set `auth_token` with a valid Polaris access token +2. **No Authentication**: For local or unsecured Polaris deployments + +## Namespace Mapping + +Polaris provides a flexible namespace hierarchy: + +- A catalog in Polaris maps to the first level Lance namespace +- Nested namespaces in Polaris map to subsequent Lance namespace levels +- Polaris supports arbitrary nesting depth, allowing flexible namespace organization + +## Table Definition + +A Lance table appears as a [Generic Table](https://github.com/polaris-catalog/polaris/blob/main/spec/polaris-catalog-apis/generic-tables-api.yaml) +object in Polaris with the following requirements: + +1. the `format` must be set to `lance` to indicate this is a Lance table +2. the `base-location` must point to the root location of the Lance table +3. the `doc` field can be used to store an optional table description +4. the `properties` must follow: + 1. there is a key `table_type` set to `lance` (case insensitive) + 2. there is a key `managed_by` set to either `storage` or `impl` (case insensitive). If not set, default to `storage` + 3. there is a key `version` set to the latest numeric version number of the table. This field will only be respected if `managed_by=impl` + +## Requirement for Implementation Managed Table + +Updates to implementation-managed Lance tables must use Polaris's table versioning mechanism +for conditional updates through the UpdateTable API. The `version` property must be updated atomically +to prevent concurrent modification conflicts. \ No newline at end of file diff --git a/java/lance-namespace-polaris/pom.xml b/java/lance-namespace-polaris/pom.xml new file mode 100644 index 000000000..bb0720601 --- /dev/null +++ b/java/lance-namespace-polaris/pom.xml @@ -0,0 +1,83 @@ + + + + 4.0.0 + + + com.lancedb + lance-namespace-root + 0.0.7 + ../pom.xml + + + lance-namespace-polaris + jar + + Lance Namespace Polaris + Polaris Catalog implementation for Lance namespace management + + + + com.lancedb + lance-namespace-core + ${lance-namespace.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + org.slf4j + slf4j-api + + + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-core + test + + + org.mockito + mockito-junit-jupiter + test + + + org.assertj + assertj-core + test + + + ch.qos.logback + logback-classic + test + + + \ No newline at end of file diff --git a/java/lance-namespace-polaris/src/main/java/com/lancedb/lance/namespace/polaris/PolarisModels.java b/java/lance-namespace-polaris/src/main/java/com/lancedb/lance/namespace/polaris/PolarisModels.java new file mode 100644 index 000000000..46ae4b659 --- /dev/null +++ b/java/lance-namespace-polaris/src/main/java/com/lancedb/lance/namespace/polaris/PolarisModels.java @@ -0,0 +1,416 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.polaris; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonNaming; + +import java.util.List; +import java.util.Map; + +/** Data transfer objects for Polaris Generic Table API. */ +public class PolarisModels { + + /** Request to create a generic table. */ + @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class CreateGenericTableRequest { + private String name; + private String format; + private String baseLocation; + private String doc; + private Map properties; + + public CreateGenericTableRequest() {} + + public CreateGenericTableRequest( + String name, + String format, + String baseLocation, + String doc, + Map properties) { + this.name = name; + this.format = format; + this.baseLocation = baseLocation; + this.doc = doc; + this.properties = properties; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public String getBaseLocation() { + return baseLocation; + } + + public void setBaseLocation(String baseLocation) { + this.baseLocation = baseLocation; + } + + public String getDoc() { + return doc; + } + + public void setDoc(String doc) { + this.doc = doc; + } + + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + } + + /** Generic table information. */ + @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) + @JsonIgnoreProperties(ignoreUnknown = true) + public static class GenericTable { + private String name; + private String format; + private String baseLocation; + private String doc; + private Map properties; + + public GenericTable() {} + + public GenericTable( + String name, + String format, + String baseLocation, + String doc, + Map properties) { + this.name = name; + this.format = format; + this.baseLocation = baseLocation; + this.doc = doc; + this.properties = properties; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public String getBaseLocation() { + return baseLocation; + } + + public void setBaseLocation(String baseLocation) { + this.baseLocation = baseLocation; + } + + public String getDoc() { + return doc; + } + + public void setDoc(String doc) { + this.doc = doc; + } + + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + } + + /** Response when loading or creating a generic table. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class LoadGenericTableResponse { + private GenericTable table; + + public LoadGenericTableResponse() {} + + public LoadGenericTableResponse(GenericTable table) { + this.table = table; + } + + public GenericTable getTable() { + return table; + } + + public void setTable(GenericTable table) { + this.table = table; + } + } + + /** Table identifier. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class TableIdentifier { + private String namespace; + private String name; + + public TableIdentifier() {} + + public TableIdentifier(String namespace, String name) { + this.namespace = namespace; + this.name = name; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + /** Response for listing generic tables. */ + @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ListGenericTablesResponse { + @JsonProperty("next-page-token") + private String nextPageToken; + + private List identifiers; + + public ListGenericTablesResponse() {} + + public ListGenericTablesResponse(String nextPageToken, List identifiers) { + this.nextPageToken = nextPageToken; + this.identifiers = identifiers; + } + + public String getNextPageToken() { + return nextPageToken; + } + + public void setNextPageToken(String nextPageToken) { + this.nextPageToken = nextPageToken; + } + + public List getIdentifiers() { + return identifiers; + } + + public void setIdentifiers(List identifiers) { + this.identifiers = identifiers; + } + } + + /** Error response from Polaris API. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class IcebergErrorResponse { + private Error error; + + public IcebergErrorResponse() {} + + public IcebergErrorResponse(Error error) { + this.error = error; + } + + public Error getError() { + return error; + } + + public void setError(Error error) { + this.error = error; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Error { + private String message; + private String type; + private int code; + private String stack; + + public Error() {} + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getStack() { + return stack; + } + + public void setStack(String stack) { + this.stack = stack; + } + } + } + + /** Namespace properties response. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class NamespaceResponse { + private List namespace; + private Map properties; + + public NamespaceResponse() {} + + public NamespaceResponse(List namespace, Map properties) { + this.namespace = namespace; + this.properties = properties; + } + + public List getNamespace() { + return namespace; + } + + public void setNamespace(List namespace) { + this.namespace = namespace; + } + + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + } + + /** List namespaces response. */ + @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ListNamespacesResponse { + @JsonProperty("next-page-token") + private String nextPageToken; + + private List namespaces; + + public ListNamespacesResponse() {} + + public ListNamespacesResponse(String nextPageToken, List namespaces) { + this.nextPageToken = nextPageToken; + this.namespaces = namespaces; + } + + public String getNextPageToken() { + return nextPageToken; + } + + public void setNextPageToken(String nextPageToken) { + this.nextPageToken = nextPageToken; + } + + public List getNamespaces() { + return namespaces; + } + + public void setNamespaces(List namespaces) { + this.namespaces = namespaces; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Namespace { + private List namespace; + + public Namespace() {} + + public Namespace(List namespace) { + this.namespace = namespace; + } + + public List getNamespace() { + return namespace; + } + + public void setNamespace(List namespace) { + this.namespace = namespace; + } + } + } + + /** Create namespace request. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CreateNamespaceRequest { + private List namespace; + private Map properties; + + public CreateNamespaceRequest() {} + + public CreateNamespaceRequest(List namespace, Map properties) { + this.namespace = namespace; + this.properties = properties; + } + + public List getNamespace() { + return namespace; + } + + public void setNamespace(List namespace) { + this.namespace = namespace; + } + + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + } +} diff --git a/java/lance-namespace-polaris/src/main/java/com/lancedb/lance/namespace/polaris/PolarisNamespace.java b/java/lance-namespace-polaris/src/main/java/com/lancedb/lance/namespace/polaris/PolarisNamespace.java new file mode 100644 index 000000000..c4ef6ca1c --- /dev/null +++ b/java/lance-namespace-polaris/src/main/java/com/lancedb/lance/namespace/polaris/PolarisNamespace.java @@ -0,0 +1,512 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.polaris; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.WriteParams; +import com.lancedb.lance.namespace.LanceNamespace; +import com.lancedb.lance.namespace.LanceNamespaceException; +import com.lancedb.lance.namespace.ObjectIdentifier; +import com.lancedb.lance.namespace.model.CreateNamespaceRequest; +import com.lancedb.lance.namespace.model.CreateNamespaceResponse; +import com.lancedb.lance.namespace.model.CreateTableRequest; +import com.lancedb.lance.namespace.model.CreateTableResponse; +import com.lancedb.lance.namespace.model.DescribeNamespaceRequest; +import com.lancedb.lance.namespace.model.DescribeNamespaceResponse; +import com.lancedb.lance.namespace.model.DescribeTableRequest; +import com.lancedb.lance.namespace.model.DescribeTableResponse; +import com.lancedb.lance.namespace.model.DropNamespaceRequest; +import com.lancedb.lance.namespace.model.DropNamespaceResponse; +import com.lancedb.lance.namespace.model.DropTableRequest; +import com.lancedb.lance.namespace.model.DropTableResponse; +import com.lancedb.lance.namespace.model.ListNamespacesRequest; +import com.lancedb.lance.namespace.model.ListNamespacesResponse; +import com.lancedb.lance.namespace.model.ListTablesRequest; +import com.lancedb.lance.namespace.model.ListTablesResponse; +import com.lancedb.lance.namespace.model.NamespaceExistsRequest; +import com.lancedb.lance.namespace.model.TableExistsRequest; +import com.lancedb.lance.namespace.rest.RestClient; +import com.lancedb.lance.namespace.util.ValidationUtil; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.Schema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Polaris Catalog namespace implementation for Lance. */ +public class PolarisNamespace implements LanceNamespace { + private static final Logger LOG = LoggerFactory.getLogger(PolarisNamespace.class); + private static final String TABLE_FORMAT_LANCE = "lance"; + private static final String MANAGED_BY_KEY = "managed_by"; + private static final String TABLE_TYPE_KEY = "table_type"; + private static final String VERSION_KEY = "version"; + + private PolarisNamespaceConfig config; + private RestClient restClient; + private BufferAllocator allocator; + + public PolarisNamespace() {} + + @Override + public void initialize(Map configProperties, BufferAllocator allocator) { + this.allocator = allocator; + this.config = new PolarisNamespaceConfig(configProperties); + + // Build REST client with authentication if provided + RestClient.Builder clientBuilder = + RestClient.builder() + .baseUrl(config.getFullApiUrl()) + .connectTimeout(config.getConnectTimeout()) + .readTimeout(config.getReadTimeout()) + .maxRetries(config.getMaxRetries()); + + // Add auth token if provided + if (config.getAuthToken() != null) { + Map headers = new HashMap<>(); + headers.put("Authorization", "Bearer " + config.getAuthToken()); + clientBuilder.defaultHeaders(headers); + } + + this.restClient = clientBuilder.build(); + LOG.info("Initialized Polaris namespace with endpoint: {}", config.getEndpoint()); + } + + @Override + public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { + ObjectIdentifier namespaceId = ObjectIdentifier.of(request.getId()); + ValidationUtil.checkArgument( + namespaceId.levels() >= 1, "Namespace must have at least one level"); + + try { + // Convert request to Polaris format + List namespace = namespaceId.listStyleId(); + + PolarisModels.CreateNamespaceRequest polarisRequest = + new PolarisModels.CreateNamespaceRequest(namespace, request.getProperties()); + + // Create namespace using Iceberg REST API endpoint + PolarisModels.NamespaceResponse response = + restClient.post("/namespaces", polarisRequest, PolarisModels.NamespaceResponse.class); + + LOG.info("Created namespace: {}", String.join(".", namespace)); + + CreateNamespaceResponse result = new CreateNamespaceResponse(); + result.setProperties(response.getProperties()); + return result; + } catch (IOException e) { + throw LanceNamespaceException.serverError( + "Failed to create namespace", "ServerError", namespaceId.stringStyleId(), e.getMessage()); + } + } + + @Override + public DescribeNamespaceResponse describeNamespace(DescribeNamespaceRequest request) { + ObjectIdentifier namespaceId = ObjectIdentifier.of(request.getId()); + ValidationUtil.checkArgument( + namespaceId.levels() >= 1, "Namespace must have at least one level"); + + try { + String namespacePath = namespaceId.stringStyleId(); + + // Get namespace properties using Iceberg REST API + PolarisModels.NamespaceResponse response = + restClient.get("/namespaces/" + namespacePath, PolarisModels.NamespaceResponse.class); + + DescribeNamespaceResponse result = new DescribeNamespaceResponse(); + result.setProperties(response.getProperties()); + return result; + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().contains("404")) { + throw LanceNamespaceException.notFound( + "Namespace not found", + "NoSuchNamespace", + namespaceId.stringStyleId(), + "Namespace not found: " + namespaceId.stringStyleId()); + } + throw LanceNamespaceException.serverError( + "Failed to describe namespace", + "ServerError", + namespaceId.stringStyleId(), + e.getMessage()); + } + } + + @Override + public ListNamespacesResponse listNamespaces(ListNamespacesRequest request) { + ObjectIdentifier parentId = + request.getId() != null + ? ObjectIdentifier.of(request.getId()) + : ObjectIdentifier.of(Collections.emptyList()); + + try { + String path = "/namespaces"; + if (!parentId.isRoot()) { + path += "/" + parentId.stringStyleId() + "/namespaces"; + } + + // List namespaces using Iceberg REST API + PolarisModels.ListNamespacesResponse response = + restClient.get(path, PolarisModels.ListNamespacesResponse.class); + + ListNamespacesResponse result = new ListNamespacesResponse(); + // Convert namespace identifiers to Set with full paths + Set namespaceSet = new LinkedHashSet<>(); + if (response.getNamespaces() != null) { + for (PolarisModels.ListNamespacesResponse.Namespace ns : response.getNamespaces()) { + namespaceSet.add(String.join(".", ns.getNamespace())); + } + } + result.setNamespaces(namespaceSet); + return result; + } catch (IOException e) { + throw LanceNamespaceException.serverError( + "Failed to list namespaces", "ServerError", "listNamespaces", e.getMessage()); + } + } + + @Override + public DropNamespaceResponse dropNamespace(DropNamespaceRequest request) { + ObjectIdentifier namespaceId = ObjectIdentifier.of(request.getId()); + ValidationUtil.checkArgument( + namespaceId.levels() >= 1, "Namespace must have at least one level"); + + try { + String namespacePath = namespaceId.stringStyleId(); + + // Drop namespace using Iceberg REST API + restClient.delete("/namespaces/" + namespacePath); + + LOG.info("Dropped namespace: {}", namespacePath); + + DropNamespaceResponse result = new DropNamespaceResponse(); + // DropNamespaceResponse has no fields to set + return result; + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().contains("404")) { + throw LanceNamespaceException.notFound( + "Namespace not found", + "NoSuchNamespace", + namespaceId.stringStyleId(), + "Namespace not found: " + namespaceId.stringStyleId()); + } + throw LanceNamespaceException.serverError( + "Failed to drop namespace", "ServerError", namespaceId.stringStyleId(), e.getMessage()); + } + } + + @Override + public void namespaceExists(NamespaceExistsRequest request) { + ObjectIdentifier namespaceId = ObjectIdentifier.of(request.getId()); + ValidationUtil.checkArgument( + namespaceId.levels() >= 1, "Namespace must have at least one level"); + + try { + String namespacePath = namespaceId.stringStyleId(); + // Use GET request to check if namespace exists + restClient.get("/namespaces/" + namespacePath, PolarisModels.NamespaceResponse.class); + // If we get here, namespace exists - return normally + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().contains("404")) { + throw LanceNamespaceException.notFound( + "Namespace not found", + "NoSuchNamespace", + namespaceId.stringStyleId(), + "Namespace not found: " + namespaceId.stringStyleId()); + } + throw LanceNamespaceException.serverError( + "Failed to check namespace existence", + "ServerError", + namespaceId.stringStyleId(), + e.getMessage()); + } + } + + @Override + public void tableExists(TableExistsRequest request) { + ObjectIdentifier tableId = ObjectIdentifier.of(request.getId()); + ValidationUtil.checkArgument( + tableId.levels() >= 2, "Table identifier must have at least 2 levels"); + + try { + // Split into namespace and table name + List parts = tableId.listStyleId(); + String tableName = parts.get(parts.size() - 1); + List namespaceParts = parts.subList(0, parts.size() - 1); + String namespacePath = String.join(".", namespaceParts); + + // Use GET request to check if table exists + restClient.get( + "/namespaces/" + namespacePath + "/generic-tables/" + tableName, + PolarisModels.LoadGenericTableResponse.class); + // If we get here, table exists - return normally + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().contains("404")) { + throw LanceNamespaceException.notFound( + "Table not found", + "NoSuchTable", + tableId.stringStyleId(), + "Table not found: " + tableId.stringStyleId()); + } + throw LanceNamespaceException.serverError( + "Failed to check table existence", + "ServerError", + tableId.stringStyleId(), + e.getMessage()); + } + } + + @Override + public CreateTableResponse createTable(CreateTableRequest request, byte[] requestData) { + ObjectIdentifier tableId = ObjectIdentifier.of(request.getId()); + ValidationUtil.checkArgument( + tableId.levels() >= 2, "Table identifier must have at least 2 levels"); + + try { + // Split into namespace and table name + List parts = tableId.listStyleId(); + String tableName = parts.get(parts.size() - 1); + List namespaceParts = parts.subList(0, parts.size() - 1); + String namespacePath = String.join(".", namespaceParts); + + // Prepare table properties + Map properties = new HashMap<>(); + String comment = null; + if (request.getProperties() != null) { + properties.putAll(request.getProperties()); + // Extract comment to use as doc field + comment = properties.remove("comment"); + } + + // Add Lance-specific properties + properties.put(TABLE_TYPE_KEY, TABLE_FORMAT_LANCE); + // Default to storage-managed unless specified + if (!properties.containsKey(MANAGED_BY_KEY)) { + properties.put(MANAGED_BY_KEY, "storage"); + } + // Only set version if managed by impl + if ("impl".equalsIgnoreCase(properties.get(MANAGED_BY_KEY))) { + properties.put(VERSION_KEY, "1"); + } + + // Create generic table request + PolarisModels.CreateGenericTableRequest tableRequest = + new PolarisModels.CreateGenericTableRequest( + tableName, + TABLE_FORMAT_LANCE, + request.getLocation(), // location from request + comment, // doc field from comment property + properties); + + // Create table using Generic Table API + PolarisModels.LoadGenericTableResponse response = + restClient.post( + "/namespaces/" + namespacePath + "/generic-tables", + tableRequest, + PolarisModels.LoadGenericTableResponse.class); + + LOG.info("Created Lance table: {}.{}", namespacePath, tableName); + + CreateTableResponse result = new CreateTableResponse(); + result.setLocation(response.getTable().getBaseLocation()); + Map resultProps = new HashMap<>(response.getTable().getProperties()); + if (response.getTable().getDoc() != null) { + resultProps.put("comment", response.getTable().getDoc()); + } + result.setProperties(resultProps); + return result; + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().contains("409")) { + throw LanceNamespaceException.conflict( + "Table already exists", + "TableAlreadyExists", + tableId.stringStyleId(), + "Table already exists: " + tableId.stringStyleId()); + } + throw LanceNamespaceException.serverError( + "Failed to create table", "ServerError", tableId.stringStyleId(), e.getMessage()); + } + } + + @Override + public DescribeTableResponse describeTable(DescribeTableRequest request) { + ObjectIdentifier tableId = ObjectIdentifier.of(request.getId()); + ValidationUtil.checkArgument( + tableId.levels() >= 2, "Table identifier must have at least 2 levels"); + + try { + // Split into namespace and table name + List parts = tableId.listStyleId(); + String tableName = parts.get(parts.size() - 1); + List namespaceParts = parts.subList(0, parts.size() - 1); + String namespacePath = String.join(".", namespaceParts); + + // Get table using Generic Table API + PolarisModels.LoadGenericTableResponse response = + restClient.get( + "/namespaces/" + namespacePath + "/generic-tables/" + tableName, + PolarisModels.LoadGenericTableResponse.class); + + PolarisModels.GenericTable table = response.getTable(); + + // Verify it's a Lance table + if (!TABLE_FORMAT_LANCE.equals(table.getFormat())) { + throw LanceNamespaceException.badRequest( + "Invalid table format", + "InvalidTableFormat", + tableId.stringStyleId(), + String.format( + "Table %s is not a Lance table (format: %s)", + tableId.stringStyleId(), table.getFormat())); + } + + DescribeTableResponse result = new DescribeTableResponse(); + result.setLocation(table.getBaseLocation()); + Map resultProps = new HashMap<>(); + if (table.getProperties() != null) { + resultProps.putAll(table.getProperties()); + } + if (table.getDoc() != null) { + resultProps.put("comment", table.getDoc()); + } + result.setProperties(resultProps); + return result; + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().contains("404")) { + throw LanceNamespaceException.notFound( + "Table not found", + "NoSuchTable", + tableId.stringStyleId(), + "Table not found: " + tableId.stringStyleId()); + } + throw LanceNamespaceException.serverError( + "Failed to describe table", "ServerError", tableId.stringStyleId(), e.getMessage()); + } + } + + @Override + public ListTablesResponse listTables(ListTablesRequest request) { + ObjectIdentifier namespaceId = ObjectIdentifier.of(request.getId()); + ValidationUtil.checkArgument( + namespaceId.levels() >= 1, "Namespace must have at least one level"); + + try { + String namespacePath = namespaceId.stringStyleId(); + + // List tables using Generic Table API + PolarisModels.ListGenericTablesResponse response = + restClient.get( + "/namespaces/" + namespacePath + "/generic-tables", + PolarisModels.ListGenericTablesResponse.class); + + ListTablesResponse result = new ListTablesResponse(); + // Convert table identifiers to table names only + Set tableNames = new LinkedHashSet<>(); + if (response.getIdentifiers() != null) { + for (PolarisModels.TableIdentifier id : response.getIdentifiers()) { + tableNames.add(id.getName()); + } + } + result.setTables(tableNames); + return result; + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().contains("404")) { + throw LanceNamespaceException.notFound( + "Namespace not found", + "NoSuchNamespace", + namespaceId.stringStyleId(), + "Namespace not found: " + namespaceId.stringStyleId()); + } + throw LanceNamespaceException.serverError( + "Failed to list tables", "ServerError", namespaceId.stringStyleId(), e.getMessage()); + } + } + + @Override + public DropTableResponse dropTable(DropTableRequest request) { + ObjectIdentifier tableId = ObjectIdentifier.of(request.getId()); + ValidationUtil.checkArgument( + tableId.levels() >= 2, "Table identifier must have at least 2 levels"); + + try { + // Split into namespace and table name + List parts = tableId.listStyleId(); + String tableName = parts.get(parts.size() - 1); + List namespaceParts = parts.subList(0, parts.size() - 1); + String namespacePath = String.join(".", namespaceParts); + + // Drop table using Generic Table API + restClient.delete("/namespaces/" + namespacePath + "/generic-tables/" + tableName); + + LOG.info("Dropped table: {}.{}", namespacePath, tableName); + + DropTableResponse result = new DropTableResponse(); + // DropTableResponse has no fields to set based on the model + return result; + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().contains("404")) { + throw LanceNamespaceException.notFound( + "Table not found", + "NoSuchTable", + tableId.stringStyleId(), + "Table not found: " + tableId.stringStyleId()); + } + throw LanceNamespaceException.serverError( + "Failed to drop table", "ServerError", tableId.stringStyleId(), e.getMessage()); + } + } + + // These methods are not part of the LanceNamespace interface + // They were removed as they don't exist in the interface + private Dataset openTableInternal(String location, Schema schema) { + try { + return Dataset.open(location, allocator); + } catch (Exception e) { + throw LanceNamespaceException.serverError( + "Failed to open Lance table", + "DatasetError", + location, + "Failed to open Lance table at: " + location + ": " + e.getMessage()); + } + } + + private Dataset createTableInternal(String location, Schema schema, WriteParams params) { + try { + return Dataset.create(allocator, location, schema, params); + } catch (Exception e) { + throw LanceNamespaceException.serverError( + "Failed to create Lance table", + "DatasetError", + location, + "Failed to create Lance table at: " + location + ": " + e.getMessage()); + } + } + + public void close() { + if (restClient != null) { + try { + restClient.close(); + } catch (IOException e) { + LOG.warn("Failed to close REST client", e); + } + } + } +} diff --git a/java/lance-namespace-polaris/src/main/java/com/lancedb/lance/namespace/polaris/PolarisNamespaceConfig.java b/java/lance-namespace-polaris/src/main/java/com/lancedb/lance/namespace/polaris/PolarisNamespaceConfig.java new file mode 100644 index 000000000..b88f6af99 --- /dev/null +++ b/java/lance-namespace-polaris/src/main/java/com/lancedb/lance/namespace/polaris/PolarisNamespaceConfig.java @@ -0,0 +1,127 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.polaris; + +import com.lancedb.lance.namespace.LanceNamespaceException; + +import java.util.Map; + +/** Configuration for Polaris namespace implementation. */ +public class PolarisNamespaceConfig { + public static final String POLARIS_ENDPOINT = "endpoint"; + public static final String POLARIS_AUTH_TOKEN = "auth_token"; + public static final String POLARIS_CONNECT_TIMEOUT = "connect_timeout"; + public static final String POLARIS_READ_TIMEOUT = "read_timeout"; + public static final String POLARIS_MAX_RETRIES = "max_retries"; + + private static final int DEFAULT_CONNECT_TIMEOUT = 10000; // 10 seconds + private static final int DEFAULT_READ_TIMEOUT = 30000; // 30 seconds + private static final int DEFAULT_MAX_RETRIES = 3; + + private final String endpoint; + private final String authToken; + private final int connectTimeout; + private final int readTimeout; + private final int maxRetries; + + public PolarisNamespaceConfig(Map properties) { + this.endpoint = getRequiredProperty(properties, POLARIS_ENDPOINT); + this.authToken = properties.get(POLARIS_AUTH_TOKEN); + this.connectTimeout = + Integer.parseInt( + properties.getOrDefault( + POLARIS_CONNECT_TIMEOUT, String.valueOf(DEFAULT_CONNECT_TIMEOUT))); + this.readTimeout = + Integer.parseInt( + properties.getOrDefault(POLARIS_READ_TIMEOUT, String.valueOf(DEFAULT_READ_TIMEOUT))); + this.maxRetries = + Integer.parseInt( + properties.getOrDefault(POLARIS_MAX_RETRIES, String.valueOf(DEFAULT_MAX_RETRIES))); + + validateConfig(); + } + + private String getRequiredProperty(Map properties, String key) { + String value = properties.get(key); + if (value == null || value.trim().isEmpty()) { + throw LanceNamespaceException.badRequest( + "Missing required configuration", + "ConfigurationError", + key, + String.format("Required configuration property '%s' is not set", key)); + } + return value.trim(); + } + + private void validateConfig() { + if (!endpoint.startsWith("http://") && !endpoint.startsWith("https://")) { + throw LanceNamespaceException.badRequest( + "Invalid endpoint format", + "ConfigurationError", + POLARIS_ENDPOINT, + "Polaris endpoint must start with http:// or https://: " + endpoint); + } + + if (connectTimeout <= 0) { + throw LanceNamespaceException.badRequest( + "Invalid timeout value", + "ConfigurationError", + POLARIS_CONNECT_TIMEOUT, + "Connect timeout must be positive: " + connectTimeout); + } + + if (readTimeout <= 0) { + throw LanceNamespaceException.badRequest( + "Invalid timeout value", + "ConfigurationError", + POLARIS_READ_TIMEOUT, + "Read timeout must be positive: " + readTimeout); + } + + if (maxRetries < 0) { + throw LanceNamespaceException.badRequest( + "Invalid retry value", + "ConfigurationError", + POLARIS_MAX_RETRIES, + "Max retries cannot be negative: " + maxRetries); + } + } + + public String getEndpoint() { + return endpoint; + } + + public String getAuthToken() { + return authToken; + } + + public int getConnectTimeout() { + return connectTimeout; + } + + public int getReadTimeout() { + return readTimeout; + } + + public int getMaxRetries() { + return maxRetries; + } + + /** Get the full API URL for Polaris catalog operations. Format: {endpoint}/api/catalog/v1 */ + public String getFullApiUrl() { + String baseUrl = + endpoint.endsWith("/") ? endpoint.substring(0, endpoint.length() - 1) : endpoint; + return baseUrl + "/api/catalog/v1"; + } +} diff --git a/java/lance-namespace-polaris/src/test/java/com/lancedb/lance/namespace/polaris/TestPolarisNamespace.java b/java/lance-namespace-polaris/src/test/java/com/lancedb/lance/namespace/polaris/TestPolarisNamespace.java new file mode 100644 index 000000000..0f9be8d6d --- /dev/null +++ b/java/lance-namespace-polaris/src/test/java/com/lancedb/lance/namespace/polaris/TestPolarisNamespace.java @@ -0,0 +1,338 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.polaris; + +import com.lancedb.lance.namespace.LanceNamespaceException; +import com.lancedb.lance.namespace.model.CreateNamespaceRequest; +import com.lancedb.lance.namespace.model.CreateNamespaceResponse; +import com.lancedb.lance.namespace.model.CreateTableRequest; +import com.lancedb.lance.namespace.model.CreateTableResponse; +import com.lancedb.lance.namespace.model.DescribeNamespaceRequest; +import com.lancedb.lance.namespace.model.DescribeNamespaceResponse; +import com.lancedb.lance.namespace.model.DescribeTableRequest; +import com.lancedb.lance.namespace.model.DescribeTableResponse; +import com.lancedb.lance.namespace.model.DropNamespaceRequest; +import com.lancedb.lance.namespace.model.DropNamespaceResponse; +import com.lancedb.lance.namespace.model.DropTableRequest; +import com.lancedb.lance.namespace.model.DropTableResponse; +import com.lancedb.lance.namespace.model.ListNamespacesRequest; +import com.lancedb.lance.namespace.model.ListNamespacesResponse; +import com.lancedb.lance.namespace.model.ListTablesRequest; +import com.lancedb.lance.namespace.model.ListTablesResponse; +import com.lancedb.lance.namespace.model.NamespaceExistsRequest; +import com.lancedb.lance.namespace.rest.RestClient; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class TestPolarisNamespace { + + @Mock private RestClient restClient; + + private PolarisNamespace namespace; + private BufferAllocator allocator; + private Map config; + + @BeforeEach + public void setUp() throws Exception { + allocator = new RootAllocator(); + namespace = new PolarisNamespace(); + + config = new HashMap<>(); + config.put("endpoint", "http://localhost:8182"); + config.put("auth_token", "test-token"); + + // Initialize namespace with config + namespace.initialize(config, allocator); + + // Replace the RestClient with mock using reflection + Field restClientField = PolarisNamespace.class.getDeclaredField("restClient"); + restClientField.setAccessible(true); + restClientField.set(namespace, restClient); + } + + @AfterEach + public void tearDown() { + if (allocator != null) { + allocator.close(); + } + } + + @Test + public void testInitialize() { + // Test successful initialization + PolarisNamespace ns = new PolarisNamespace(); + ns.initialize(config, allocator); + // Should not throw + } + + @Test + public void testInitializeMissingEndpoint() { + Map badConfig = new HashMap<>(); + badConfig.put("polaris.catalog", "test_catalog"); + + PolarisNamespace ns = new PolarisNamespace(); + assertThatThrownBy(() -> ns.initialize(badConfig, allocator)) + .isInstanceOf(LanceNamespaceException.class) + .hasMessageContaining("Required configuration property 'endpoint' is not set"); + } + + @Test + public void testCreateNamespace() throws IOException { + PolarisModels.NamespaceResponse mockResponse = new PolarisModels.NamespaceResponse(); + mockResponse.setNamespace(Arrays.asList("test_catalog", "schema1")); + mockResponse.setProperties(Collections.singletonMap("key", "value")); + + when(restClient.post( + eq("/namespaces"), + any(PolarisModels.CreateNamespaceRequest.class), + eq(PolarisModels.NamespaceResponse.class))) + .thenReturn(mockResponse); + + CreateNamespaceRequest request = new CreateNamespaceRequest(); + request.setId(Arrays.asList("test_catalog", "schema1")); + request.setProperties(Collections.singletonMap("key", "value")); + + CreateNamespaceResponse response = namespace.createNamespace(request); + + // Response doesn't have getId() method, just verify properties + assertThat(response.getProperties()).containsEntry("key", "value"); + } + + @Test + public void testDescribeNamespace() throws IOException { + PolarisModels.NamespaceResponse mockResponse = new PolarisModels.NamespaceResponse(); + mockResponse.setNamespace(Arrays.asList("test_catalog", "schema1")); + mockResponse.setProperties(Collections.singletonMap("description", "test schema")); + + when(restClient.get( + eq("/namespaces/test_catalog.schema1"), eq(PolarisModels.NamespaceResponse.class))) + .thenReturn(mockResponse); + + DescribeNamespaceRequest request = new DescribeNamespaceRequest(); + request.setId(Arrays.asList("test_catalog", "schema1")); + + DescribeNamespaceResponse response = namespace.describeNamespace(request); + + // Response doesn't have getId() method, just verify properties + assertThat(response.getProperties()).containsEntry("description", "test schema"); + } + + @Test + public void testListNamespaces() throws IOException { + PolarisModels.ListNamespacesResponse mockResponse = new PolarisModels.ListNamespacesResponse(); + PolarisModels.ListNamespacesResponse.Namespace ns1 = + new PolarisModels.ListNamespacesResponse.Namespace(); + ns1.setNamespace(Arrays.asList("test_catalog", "schema1")); + PolarisModels.ListNamespacesResponse.Namespace ns2 = + new PolarisModels.ListNamespacesResponse.Namespace(); + ns2.setNamespace(Arrays.asList("test_catalog", "schema2")); + mockResponse.setNamespaces(Arrays.asList(ns1, ns2)); + + when(restClient.get(eq("/namespaces"), eq(PolarisModels.ListNamespacesResponse.class))) + .thenReturn(mockResponse); + + ListNamespacesRequest request = new ListNamespacesRequest(); + + ListNamespacesResponse response = namespace.listNamespaces(request); + + assertThat(response.getNamespaces()).hasSize(2); + assertThat(response.getNamespaces()).contains("test_catalog.schema1", "test_catalog.schema2"); + } + + @Test + public void testDropNamespace() throws IOException { + DropNamespaceRequest request = new DropNamespaceRequest(); + request.setId(Arrays.asList("test_catalog", "schema1")); + + DropNamespaceResponse response = namespace.dropNamespace(request); + + verify(restClient).delete("/namespaces/test_catalog.schema1"); + // Response doesn't have getId() method, just verify the delete was called + } + + @Test + public void testNamespaceExists() throws IOException { + when(restClient.get( + eq("/namespaces/test_catalog.schema1"), eq(PolarisModels.NamespaceResponse.class))) + .thenReturn(new PolarisModels.NamespaceResponse()); + + NamespaceExistsRequest request = new NamespaceExistsRequest(); + request.setId(Arrays.asList("test_catalog", "schema1")); + + // namespaceExists returns void - it throws exception if not exists + namespace.namespaceExists(request); // Should not throw + } + + @Test + public void testNamespaceNotExists() throws IOException { + when(restClient.get( + eq("/namespaces/test_catalog.schema1"), eq(PolarisModels.NamespaceResponse.class))) + .thenThrow(new IOException("404 Not Found")); + + NamespaceExistsRequest request = new NamespaceExistsRequest(); + request.setId(Arrays.asList("test_catalog", "schema1")); + + // namespaceExists should throw when namespace doesn't exist + assertThatThrownBy(() -> namespace.namespaceExists(request)) + .isInstanceOf(LanceNamespaceException.class); + } + + @Test + public void testCreateTable() throws IOException { + PolarisModels.GenericTable mockTable = new PolarisModels.GenericTable(); + mockTable.setName("test_table"); + mockTable.setFormat("lance"); + mockTable.setBaseLocation("s3://bucket/path/to/table"); + mockTable.setDoc("Test table"); // Should be returned in doc field + Map props = new HashMap<>(); + props.put("table_type", "lance"); + mockTable.setProperties(props); + + PolarisModels.LoadGenericTableResponse mockResponse = + new PolarisModels.LoadGenericTableResponse(); + mockResponse.setTable(mockTable); + + when(restClient.post( + eq("/namespaces/test_catalog.schema1/generic-tables"), + any(PolarisModels.CreateGenericTableRequest.class), + eq(PolarisModels.LoadGenericTableResponse.class))) + .thenReturn(mockResponse); + + CreateTableRequest request = new CreateTableRequest(); + request.setId(Arrays.asList("test_catalog", "schema1", "test_table")); + request.setLocation("s3://bucket/path/to/table"); + request.setProperties(Collections.singletonMap("comment", "Test table")); + + CreateTableResponse response = namespace.createTable(request, new byte[0]); + + assertThat(response.getLocation()).isEqualTo("s3://bucket/path/to/table"); + assertThat(response.getProperties()).containsEntry("table_type", "lance"); + assertThat(response.getProperties()).containsEntry("comment", "Test table"); + } + + @Test + public void testDescribeTable() throws IOException { + PolarisModels.GenericTable mockTable = new PolarisModels.GenericTable(); + mockTable.setName("test_table"); + mockTable.setFormat("lance"); + mockTable.setBaseLocation("s3://bucket/path/to/table"); + mockTable.setDoc("Test table"); + Map props = new HashMap<>(); + props.put("table_type", "lance"); + mockTable.setProperties(props); + + PolarisModels.LoadGenericTableResponse mockResponse = + new PolarisModels.LoadGenericTableResponse(); + mockResponse.setTable(mockTable); + + when(restClient.get( + eq("/namespaces/test_catalog.schema1/generic-tables/test_table"), + eq(PolarisModels.LoadGenericTableResponse.class))) + .thenReturn(mockResponse); + + DescribeTableRequest request = new DescribeTableRequest(); + request.setId(Arrays.asList("test_catalog", "schema1", "test_table")); + + DescribeTableResponse response = namespace.describeTable(request); + + assertThat(response.getLocation()).isEqualTo("s3://bucket/path/to/table"); + assertThat(response.getProperties()).containsEntry("comment", "Test table"); + assertThat(response.getProperties()).containsEntry("table_type", "lance"); + } + + @Test + public void testDescribeTableNotLanceFormat() throws IOException { + PolarisModels.GenericTable mockTable = new PolarisModels.GenericTable(); + mockTable.setName("test_table"); + mockTable.setFormat("iceberg"); // Not a Lance table + mockTable.setBaseLocation("s3://bucket/path/to/table"); + + PolarisModels.LoadGenericTableResponse mockResponse = + new PolarisModels.LoadGenericTableResponse(); + mockResponse.setTable(mockTable); + + when(restClient.get( + eq("/namespaces/test_catalog.schema1/generic-tables/test_table"), + eq(PolarisModels.LoadGenericTableResponse.class))) + .thenReturn(mockResponse); + + DescribeTableRequest request = new DescribeTableRequest(); + request.setId(Arrays.asList("test_catalog", "schema1", "test_table")); + + assertThatThrownBy(() -> namespace.describeTable(request)) + .isInstanceOf(LanceNamespaceException.class) + .hasMessageContaining("is not a Lance table"); + } + + @Test + public void testListTables() throws IOException { + PolarisModels.TableIdentifier id1 = new PolarisModels.TableIdentifier(); + id1.setNamespace("test_catalog.schema1"); + id1.setName("table1"); + + PolarisModels.TableIdentifier id2 = new PolarisModels.TableIdentifier(); + id2.setNamespace("test_catalog.schema1"); + id2.setName("table2"); + + PolarisModels.ListGenericTablesResponse mockResponse = + new PolarisModels.ListGenericTablesResponse(); + mockResponse.setIdentifiers(Arrays.asList(id1, id2)); + + when(restClient.get( + eq("/namespaces/test_catalog.schema1/generic-tables"), + eq(PolarisModels.ListGenericTablesResponse.class))) + .thenReturn(mockResponse); + + ListTablesRequest request = new ListTablesRequest(); + request.setId(Arrays.asList("test_catalog", "schema1")); + + ListTablesResponse response = namespace.listTables(request); + + assertThat(response.getTables()).hasSize(2); + assertThat(response.getTables()).contains("table1", "table2"); + } + + @Test + public void testDropTable() throws IOException { + DropTableRequest request = new DropTableRequest(); + request.setId(Arrays.asList("test_catalog", "schema1", "test_table")); + + DropTableResponse response = namespace.dropTable(request); + + verify(restClient).delete("/namespaces/test_catalog.schema1/generic-tables/test_table"); + // Response doesn't have getId() method, just verify the delete was called + } +} diff --git a/java/lance-namespace-polaris/src/test/java/com/lancedb/lance/namespace/polaris/TestPolarisNamespaceIntegration.java b/java/lance-namespace-polaris/src/test/java/com/lancedb/lance/namespace/polaris/TestPolarisNamespaceIntegration.java new file mode 100644 index 000000000..0335f9d7c --- /dev/null +++ b/java/lance-namespace-polaris/src/test/java/com/lancedb/lance/namespace/polaris/TestPolarisNamespaceIntegration.java @@ -0,0 +1,310 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.namespace.polaris; + +import com.lancedb.lance.namespace.LanceNamespaceException; +import com.lancedb.lance.namespace.model.CreateNamespaceRequest; +import com.lancedb.lance.namespace.model.CreateNamespaceResponse; +import com.lancedb.lance.namespace.model.CreateTableRequest; +import com.lancedb.lance.namespace.model.CreateTableResponse; +import com.lancedb.lance.namespace.model.DescribeNamespaceRequest; +import com.lancedb.lance.namespace.model.DescribeNamespaceResponse; +import com.lancedb.lance.namespace.model.DescribeTableRequest; +import com.lancedb.lance.namespace.model.DescribeTableResponse; +import com.lancedb.lance.namespace.model.DropNamespaceRequest; +import com.lancedb.lance.namespace.model.DropTableRequest; +import com.lancedb.lance.namespace.model.ListNamespacesRequest; +import com.lancedb.lance.namespace.model.ListNamespacesResponse; +import com.lancedb.lance.namespace.model.ListTablesRequest; +import com.lancedb.lance.namespace.model.ListTablesResponse; +import com.lancedb.lance.namespace.model.NamespaceExistsRequest; +import com.lancedb.lance.namespace.model.TableExistsRequest; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Integration tests for PolarisNamespace against a running Polaris instance. + * + *

To run these tests, start Polaris with: - Endpoint: http://localhost:8182 - Credentials: + * CLIENT_ID=root, CLIENT_SECRET=s3cr3t + * + *

Tests are automatically skipped if Polaris is not available. + */ +public class TestPolarisNamespaceIntegration { + + private static final String POLARIS_ENDPOINT = "http://localhost:8182"; + private static final String CLIENT_ID = "root"; + private static final String CLIENT_SECRET = "s3cr3t"; + private static boolean polarisAvailable = false; + + private PolarisNamespace namespace; + private BufferAllocator allocator; + private String testCatalog; + private String testNamespace; + + @BeforeAll + public static void checkPolarisAvailable() { + try { + // Try to check if Polaris API is available by checking a known endpoint + // We'll try the namespaces endpoint which should exist + URL url = new URL(POLARIS_ENDPOINT + "/api/catalog/v1/namespaces"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout(1000); + conn.setReadTimeout(1000); + + // Add basic auth header to check if we can authenticate + String auth = + java.util.Base64.getEncoder() + .encodeToString((CLIENT_ID + ":" + CLIENT_SECRET).getBytes()); + conn.setRequestProperty("Authorization", "Basic " + auth); + + int responseCode = conn.getResponseCode(); + conn.disconnect(); + + // Consider Polaris available if we get any HTTP response (even 401/403) + // but not 404 which means the endpoint doesn't exist + polarisAvailable = responseCode != 404 && responseCode > 0; + + if (!polarisAvailable) { + System.out.println( + "Polaris is not available at " + POLARIS_ENDPOINT + " - skipping integration tests"); + } else { + System.out.println( + "Polaris detected at " + POLARIS_ENDPOINT + " (response code: " + responseCode + ")"); + } + } catch (Exception e) { + polarisAvailable = false; + System.out.println( + "Polaris is not available at " + + POLARIS_ENDPOINT + + " (" + + e.getMessage() + + ") - skipping integration tests"); + } + } + + @BeforeEach + public void setUp() throws Exception { + // Skip all tests if Polaris is not available + Assumptions.assumeTrue(polarisAvailable, "Polaris is not available at " + POLARIS_ENDPOINT); + + allocator = new RootAllocator(); + namespace = new PolarisNamespace(); + + // Generate unique names for this test run + String uniqueId = UUID.randomUUID().toString().substring(0, 8); + testCatalog = "test_catalog"; // Use a fixed catalog that should exist + testNamespace = "test_ns_" + uniqueId; + + Map config = new HashMap<>(); + config.put("endpoint", POLARIS_ENDPOINT); + + // Try to get OAuth token first + String token = getOAuthToken(); + if (token != null) { + config.put("auth_token", token); + } else { + // Fall back to basic auth + String basicAuth = + java.util.Base64.getEncoder() + .encodeToString((CLIENT_ID + ":" + CLIENT_SECRET).getBytes()); + config.put("auth_token", basicAuth); + } + + namespace.initialize(config, allocator); + } + + @AfterEach + public void tearDown() { + // Clean up test resources + try { + // Drop test namespace if it exists + DropNamespaceRequest dropRequest = new DropNamespaceRequest(); + dropRequest.setId(Arrays.asList(testCatalog, testNamespace)); + namespace.dropNamespace(dropRequest); + } catch (Exception e) { + // Ignore cleanup errors + } + + if (allocator != null) { + allocator.close(); + } + } + + private String getOAuthToken() { + try { + // Attempt to get OAuth token + URL url = new URL(POLARIS_ENDPOINT + "/api/catalog/v1/oauth/tokens"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + conn.setDoOutput(true); + + String params = + String.format( + "grant_type=client_credentials&client_id=%s&client_secret=%s&scope=PRINCIPAL_ROLE:ALL", + CLIENT_ID, CLIENT_SECRET); + + conn.getOutputStream().write(params.getBytes()); + + if (conn.getResponseCode() == 200) { + // Parse token from response (simplified - in production use proper JSON parsing) + java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int len; + while ((len = conn.getInputStream().read(buffer)) != -1) { + baos.write(buffer, 0, len); + } + String responseStr = new String(baos.toByteArray()); + // Extract token from JSON response + int tokenStart = responseStr.indexOf("\"access_token\":\"") + 16; + if (tokenStart > 16) { + int tokenEnd = responseStr.indexOf("\"", tokenStart); + return responseStr.substring(tokenStart, tokenEnd); + } + } + } catch (Exception e) { + // OAuth not available, will fall back to basic auth + } + return null; + } + + @Test + public void testNamespaceOperations() { + // Create namespace + CreateNamespaceRequest createRequest = new CreateNamespaceRequest(); + createRequest.setId(Arrays.asList(testCatalog, testNamespace)); + createRequest.setProperties(Collections.singletonMap("description", "Test namespace")); + + CreateNamespaceResponse createResponse = namespace.createNamespace(createRequest); + // Response doesn't have getId() method, just verify properties + assertThat(createResponse.getProperties()).containsEntry("description", "Test namespace"); + + // Describe namespace + DescribeNamespaceRequest describeRequest = new DescribeNamespaceRequest(); + describeRequest.setId(Arrays.asList(testCatalog, testNamespace)); + + DescribeNamespaceResponse describeResponse = namespace.describeNamespace(describeRequest); + // Response doesn't have getId() method, just verify we got a response + assertThat(describeResponse).isNotNull(); + + // Check namespace exists + NamespaceExistsRequest existsRequest = new NamespaceExistsRequest(); + existsRequest.setId(Arrays.asList(testCatalog, testNamespace)); + namespace.namespaceExists(existsRequest); // Should not throw + + // List namespaces + ListNamespacesRequest listRequest = new ListNamespacesRequest(); + ListNamespacesResponse listResponse = namespace.listNamespaces(listRequest); + assertThat(listResponse.getNamespaces()) + .anyMatch(ns -> ns.equals(Arrays.asList(testCatalog, testNamespace))); + + // Drop namespace + DropNamespaceRequest dropRequest = new DropNamespaceRequest(); + dropRequest.setId(Arrays.asList(testCatalog, testNamespace)); + namespace.dropNamespace(dropRequest); + + // Verify namespace doesn't exist + assertThatThrownBy(() -> namespace.namespaceExists(existsRequest)) + .isInstanceOf(LanceNamespaceException.class) + .hasMessageContaining("404"); + } + + @Test + public void testTableOperations() { + // Create namespace first + CreateNamespaceRequest nsRequest = new CreateNamespaceRequest(); + nsRequest.setId(Arrays.asList(testCatalog, testNamespace)); + namespace.createNamespace(nsRequest); + + String tableName = "test_table_" + UUID.randomUUID().toString().substring(0, 8); + + // Create table + CreateTableRequest createRequest = new CreateTableRequest(); + createRequest.setId(Arrays.asList(testCatalog, testNamespace, tableName)); + createRequest.setLocation("s3://test-bucket/lance/" + tableName); + createRequest.setProperties(Collections.singletonMap("comment", "Test table")); + + CreateTableResponse createResponse = namespace.createTable(createRequest, new byte[0]); + assertThat(createResponse.getLocation()).isEqualTo("s3://test-bucket/lance/" + tableName); + assertThat(createResponse.getProperties()).containsEntry("comment", "Test table"); + + // Describe table + DescribeTableRequest describeRequest = new DescribeTableRequest(); + describeRequest.setId(Arrays.asList(testCatalog, testNamespace, tableName)); + + DescribeTableResponse describeResponse = namespace.describeTable(describeRequest); + assertThat(describeResponse.getLocation()).isEqualTo("s3://test-bucket/lance/" + tableName); + + // Check table exists + TableExistsRequest existsRequest = new TableExistsRequest(); + existsRequest.setId(Arrays.asList(testCatalog, testNamespace, tableName)); + namespace.tableExists(existsRequest); // Should not throw + + // List tables + ListTablesRequest listRequest = new ListTablesRequest(); + listRequest.setId(Arrays.asList(testCatalog, testNamespace)); + + ListTablesResponse listResponse = namespace.listTables(listRequest); + assertThat(listResponse.getTables()).contains(tableName); + + // Drop table + DropTableRequest dropRequest = new DropTableRequest(); + dropRequest.setId(Arrays.asList(testCatalog, testNamespace, tableName)); + namespace.dropTable(dropRequest); + + // Verify table doesn't exist + assertThatThrownBy(() -> namespace.tableExists(existsRequest)) + .isInstanceOf(LanceNamespaceException.class) + .hasMessageContaining("404"); + } + + @Test + public void testCreateTableWithInvalidFormat() { + // Create namespace first + CreateNamespaceRequest nsRequest = new CreateNamespaceRequest(); + nsRequest.setId(Arrays.asList(testCatalog, testNamespace)); + namespace.createNamespace(nsRequest); + + // Try to describe a non-Lance table (would need to be created through Polaris directly) + // This test demonstrates the format validation + + // For now, just verify Lance tables work correctly + String tableName = "lance_table"; + CreateTableRequest createRequest = new CreateTableRequest(); + createRequest.setId(Arrays.asList(testCatalog, testNamespace, tableName)); + createRequest.setLocation("s3://test-bucket/lance/" + tableName); + + CreateTableResponse response = namespace.createTable(createRequest, new byte[0]); + assertThat(response.getProperties()).containsEntry("table_type", "lance"); + } +} diff --git a/java/pom.xml b/java/pom.xml index 96e8c4002..65ec09325 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -95,6 +95,7 @@ lance-namespace-hive3 lance-namespace-glue lance-namespace-unity + lance-namespace-polaris lance-namespace-lancedb