feat: support create, describe and drop table for Hive and refactor Hive2 and Hive3 (#166)

Author: jackye1995

docs/src/spec/concepts.md

@@ -42,38 +42,45 @@ The identifier of any object must be unique among all other objects that share t
 
 Based on the uniqueness property of an object name within its parent namespace,
 an object identifier is the list of object names starting from (not including) the root namespace to (including) the object itself.
-This is also called an **list identifier**.
+This is also called an **list style identifier**.
 For examples:
 
-- the list identifier of `cat5` is `[cat2, cat5]`
-- the list identifier of `t1` is `[cat2, cat5, t1]`
+- the list style identifier of `cat5` is `[cat2, cat5]`
+- the list style identifier of `t1` is `[cat2, cat5, t1]`
 
-The dot (`.`) symbol is typically used as the delimiter to join all the names to form an **string identifier**,
+The dot (`.`) symbol is typically used as the delimiter to join all the names to form an **string style identifier**,
 but other symbols could also be used if dot is used in the object name.
 For examples:
 
-- the string identifier of `cat5` is `cat2.cat5`
-- the string identifier of `t1` is `cat2.cat5.t1`
-- the string identifier of `t3` is `cat4$t3` when using delimiter `$`
+- the string style identifier of `cat5` is `cat2.cat5`
+- the string style identifier of `t1` is `cat2.cat5.t1`
+- the string style identifier of `t3` is `cat4$t3` when using delimiter `$`
 
 ## Name and Identifier for Root Namespace
 
 The root namespace itself has no name or identifier.
-When represented in code, its name and string identifier is represented by an empty or null string,
-and its list identifier is represented by an empty or null list.
+When represented in code, its name and string style identifier is represented by an empty or null string,
+and its list style identifier is represented by an empty or null list.
 
 The actual name and identifier of the root namespace is typically
 assigned by users through some configuration when used in a tool.
 For example, a root namespace can be called `cat1` in Ray, but called `cat2` in Apache Spark,
 and they are both configured to connect to the same root namespace.
 
-## Namespace Level
+## Object Level
 
-If every table has the same number of namespaces all the way to the root namespace,
-the namespace is called **leveled**. The [example above](#namespace-definition) is not leveled
-because `t1` has 2 namespaces `ns1` and `ns4` before root, whereas `t2` has 1 namespace `ns2` before root.
+The root namespace is always at level 0.
+This means if an object has list style identifier with list size `N`, 
+the object is at the `N`th level in the entire namespace hierarchy.
+We also say the object identifier has `N` levels.
+For examples, a namespace `[ns1, ns2]` is at level 2, the identifier `ns1.ns2` has 2 levels.
+A table `[catalog1, database2, table3]` is at level 3, the identifier `catalog1.database2.table3` has 3 levels.
+
+### Leveled Namespace
 
-For a leveled namespace, the number of namespaces up to and including the root for any table 
-is referred to as the **number of levels**.
+If every table in the root namespace are at the same level `N`, the namespace is called **leveled**,
+and we say this namespace is a `N`-level namespace.
 For example, a [directory namespace](../impls/dir) is a 1-level namespace,
-and a [Hive 2.x namespace](../impls/hive) is a 2-level namespace.
+and a [Hive 2.x namespace](../impls/hive) is a 2-level namespace.
+The [example above](#namespace-definition) is not leveled
+because `t1` has 2 namespaces `ns1` and `ns4` before root, whereas `t2` has 1 namespace `ns2` before root.

java/Makefile

@@ -51,6 +51,8 @@ gen-springboot-server: clean-springboot-server
       --additional-properties=groupId=com.lancedb,artifactId=lance-namespace-springboot-server,artifactVersion=$(VERSION),parentGroupId=com.lancedb,parentArtifactId=lance-namespace-root,parentVersion=$(VERSION),parentRelativePath=pom.xml,library=spring-boot,interfaceOnly=true,useOptional=true,openApiNullable=false,java8=true,apiPackage=com.lancedb.lance.namespace.server.springboot.api,modelPackage=com.lancedb.lance.namespace.server.springboot.model,useTags=true,skipDefaultInterface=false,hideGenerationTimestamp=true,licenseName=Apache-2.0,licenseUrl=https://www.apache.org/licenses/LICENSE-2.0.txt
 	rm -rf lance-namespace-springboot-server/.openapi-generator-ignore
 	rm -rf lance-namespace-springboot-server/.openapi-generator
+	rm -rf lance-namespace-springboot-server/pom.xml
+	cp springboot-server-pom.xml lance-namespace-springboot-server/pom.xml
 
 .PHONY: lint-springboot-server
 lint-springboot-server: gen-apache-client gen-springboot-server
@@ -70,10 +72,15 @@ build-adapter:
 	./mvnw spotless:apply -pl lance-namespace-adapter -am
 	./mvnw install -pl lance-namespace-adapter -am
 
-.PHONY: build-hive
-build-hive:
-	./mvnw spotless:apply -pl lance-namespace-hive -am
-	./mvnw install -pl lance-namespace-hive -am
+.PHONY: build-hive2
+build-hive2:
+	./mvnw spotless:apply -pl lance-namespace-hive2 -am
+	./mvnw install -pl lance-namespace-hive2 -am
+
+.PHONY: build-hive3
+build-hive3:
+	./mvnw spotless:apply -pl lance-namespace-hive3 -am
+	./mvnw install -pl lance-namespace-hive3 -am
 
 .PHONY: build-glue
 build-glue:
@@ -92,4 +99,4 @@ clean: clean-apache-client clean-springboot-server
 gen: gen-apache-client gen-springboot-server lint-apache-client lint-springboot-server
 
 .PHONY: build
-build: build-apache-client build-springboot-server build-core build-adapter build-hive build-glue build-lancedb
+build: build-apache-client build-springboot-server build-core build-adapter build-hive2 build-hive3 build-glue build-lancedb

java/lance-namespace-adapter/src/main/java/com/lancedb/lance/namespace/adapter/ServerToClientRequest.java

@@ -77,7 +77,7 @@ public static DescribeNamespaceRequest describeNamespace(
   public static ListNamespacesRequest listNamespaces(
       String id, Optional<String> delimiter, Optional<String> pageToken, Optional<Integer> limit) {
     ListNamespacesRequest converted = new ListNamespacesRequest();
-    converted.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).idListStyle());
+    converted.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).listStyleId());
     converted.setPageToken(pageToken.orElse(null));
     converted.setLimit(limit.orElse(null));
     return converted;
@@ -86,7 +86,7 @@ public static ListNamespacesRequest listNamespaces(
   public static ListTablesRequest listTables(
       String id, Optional<String> delimiter, Optional<String> pageToken, Optional<Integer> limit) {
     ListTablesRequest converted = new ListTablesRequest();
-    converted.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).idListStyle());
+    converted.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).listStyleId());
     converted.setPageToken(pageToken.orElse(null));
     converted.setLimit(limit.orElse(null));
     return converted;

java/lance-namespace-adapter/src/main/java/com/lancedb/lance/namespace/adapter/TableController.java

@@ -118,7 +118,7 @@ public ResponseEntity<CreateTableResponse> createTable(
       byte[] data = readAllBytes(body.getInputStream());
       com.lancedb.lance.namespace.server.springboot.model.CreateTableRequest request =
           new com.lancedb.lance.namespace.server.springboot.model.CreateTableRequest();
-      request.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).idListStyle());
+      request.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).listStyleId());
       xLanceTableLocation.ifPresent(request::setLocation);
       if (xLanceTableProperties.isPresent()) {
         // Parse JSON properties
@@ -177,7 +177,7 @@ public ResponseEntity<InsertIntoTableResponse> insertIntoTable(
       byte[] data = readAllBytes(body.getInputStream());
       com.lancedb.lance.namespace.server.springboot.model.InsertIntoTableRequest request =
           new com.lancedb.lance.namespace.server.springboot.model.InsertIntoTableRequest();
-      request.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).idListStyle());
+      request.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).listStyleId());
       request.setMode(
           com.lancedb.lance.namespace.server.springboot.model.InsertIntoTableRequest.ModeEnum
               .fromValue(mode.orElse("append")));
@@ -213,7 +213,7 @@ public ResponseEntity<MergeInsertIntoTableResponse> mergeInsertIntoTable(
       byte[] data = readAllBytes(body.getInputStream());
       com.lancedb.lance.namespace.server.springboot.model.MergeInsertIntoTableRequest request =
           new com.lancedb.lance.namespace.server.springboot.model.MergeInsertIntoTableRequest();
-      request.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).idListStyle());
+      request.setId(ObjectIdentifier.of(id, delimiter.orElse(null)).listStyleId());
       request.setOn(on);
       request.setWhenMatchedUpdateAll(whenMatchedUpdateAll.orElse(false));
       whenMatchedUpdateAllFilt.ifPresent(request::setWhenMatchedUpdateAllFilt);

java/lance-namespace-core/pom.xml

@@ -90,6 +90,18 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>${maven-surefire-plugin.version}</version>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>3.3.0</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
         <extensions>
             <extension>

java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/LanceNamespaces.java

@@ -26,7 +26,8 @@ public class LanceNamespaces {
       ImmutableMap.<String, String>builder()
           .put("dir", "com.lancedb.lance.namespace.dir.DirectoryNamespace")
           .put("rest", "com.lancedb.lance.namespace.rest.RestNamespace")
-          .put("hive", "com.lancedb.lance.namespace.hive.HiveNamespace")
+          .put("hive2", "com.lancedb.lance.namespace.hive2.Hive2Namespace")
+          .put("hive3", "com.lancedb.lance.namespace.hive3.Hive3Namespace")
           .put("glue", "com.lancedb.lance.namespace.glue.GlueNamespace")
           .build();
 

java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/ObjectIdentifier.java

@@ -37,7 +37,7 @@ public static ObjectIdentifier of(List<String> levels) {
         .forEach(
             level ->
                 ValidationUtil.checkNotNullOrEmptyString(
-                    level, "Invalid namespace containing empty string %s", levels));
+                    level, "Invalid namespace containing empty string levels %s", levels));
     return new ObjectIdentifier(levels.toArray(new String[0]));
   }
 
@@ -54,22 +54,22 @@ public static ObjectIdentifier of(String id, String delimiter) {
     return new ObjectIdentifier(id.split(delimiter));
   }
 
-  public String level(int pos) {
+  public String levelAtListPos(int pos) {
     return levels[pos];
   }
 
-  public String idStringStyle() {
-    return idStringStyle(DELIMITER_DEFAULT);
+  public String stringStyleId() {
+    return stringStyleId(DELIMITER_DEFAULT);
   }
 
-  public String idStringStyle(String delimiter) {
+  public String stringStyleId(String delimiter) {
     if (levels.length == 0) {
       return delimiter;
     }
     return String.join(delimiter, levels);
   }
 
-  public List<String> idListStyle() {
+  public List<String> listStyleId() {
     return ImmutableList.copyOf(levels);
   }
 
@@ -82,7 +82,7 @@ public List<String> parent() {
   }
 
   public int levels() {
-    return levels.length + 1;
+    return levels.length;
   }
 
   public boolean isRoot() {