Skip to content

Commit c6764ed

Browse files
wojiaodoubaolijinglun
wojiaodoubao
and
lijinglun
authored
feat: add hive create namespace (#154)
Close #156 Related to #92 --------- Co-authored-by: lijinglun <lijinglun@bytedance.com>
1 parent 737cf82 commit c6764ed

13 files changed

Lines changed: 730 additions & 51 deletions

File tree

‎java/lance-namespace-adapter/src/test/java/com/lancedb/lance/namespace/adapter/MockExceptionController.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@
2323
public class MockExceptionController {
2424
@GetMapping("/testNotFound")
2525
public String testNotFound(@RequestParam(required = false) String param) {
26-
String type = "Mock resource not found";
27-
String error = "Not found Error";
26+
String error = "Mock resource not found";
27+
String type = "Not found Error";
2828
String instance = "/v1/namespaces";
2929
String detail = String.format("%s not found", param);
30-
throw LanceNamespaceException.notFound(type, error, instance, detail);
30+
throw LanceNamespaceException.notFound(error, type, instance, detail);
3131
}
3232

3333
@GetMapping("/testInternalError")

‎java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/util/CommonUtil.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.lancedb.lance.namespace.util;
15+
16+
import com.google.common.base.Joiner;
17+
18+
import java.net.URI;
19+
import java.net.URISyntaxException;
20+
import java.util.Arrays;
21+
22+
public class CommonUtil {
23+
private CommonUtil() {}
24+
25+
public static String formatCurrentStackTrace() {
26+
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
27+
return formatStackTrace(stackTrace);
28+
}
29+
30+
public static String formatStackTrace(StackTraceElement[] stackTrace) {
31+
return Joiner.on("\n\t").join(Arrays.copyOfRange(stackTrace, 1, stackTrace.length));
32+
}
33+
34+
/**
35+
* Parse absolute path string into qualified path. Qualified path format is
36+
* '{scheme}://{authority}/{path}'.
37+
*/
38+
public static String makeQualified(String absolutePath) {
39+
try {
40+
URI uri = new URI(absolutePath);
41+
ValidationUtil.checkArgument(
42+
uri.isAbsolute(), "Couldn't parse %s because it is not absolute.", absolutePath);
43+
44+
String scheme = uri.getScheme();
45+
String authority = uri.getAuthority() == null ? "" : uri.getAuthority();
46+
String path = uri.getPath();
47+
return String.format("%s://%s%s", scheme, authority, path);
48+
} catch (URISyntaxException e) {
49+
throw new IllegalArgumentException(String.format("Invalid path %s", absolutePath), e);
50+
}
51+
}
52+
}

‎java/lance-namespace-core/src/main/java/com/lancedb/lance/namespace/util/ValidationUtil.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ public static void checkNotNull(Object object, String message, Object... args) {
2828
checkArgument(object != null, message, args);
2929
}
3030

31-
public static void checkNotNullOrEmptyString(String str, String message, Object... args) {
31+
public static String checkNotNullOrEmptyString(String str, String message, Object... args) {
3232
checkArgument(str != null && !str.isEmpty(), message, args);
33+
return str;
3334
}
3435

3536
public static void checkState(boolean expression, String message, Object... args) {

‎java/lance-namespace-core/src/test/java/com/lancedb/lance/namespace/TestCommonUtil.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.lancedb.lance.namespace;
15+
16+
import com.lancedb.lance.namespace.util.CommonUtil;
17+
18+
import org.junit.jupiter.api.Test;
19+
20+
import static org.junit.jupiter.api.Assertions.assertEquals;
21+
import static org.junit.jupiter.api.Assertions.assertThrows;
22+
23+
public class TestCommonUtil {
24+
@Test
25+
public void testMakeQualified() {
26+
// Case 1: Un-absolute path
27+
assertThrows(IllegalArgumentException.class, () -> CommonUtil.makeQualified("./cat"));
28+
assertThrows(IllegalArgumentException.class, () -> CommonUtil.makeQualified("cat"));
29+
assertThrows(IllegalArgumentException.class, () -> CommonUtil.makeQualified("/cat"));
30+
31+
// Case 2: null and empty
32+
assertThrows(NullPointerException.class, () -> CommonUtil.makeQualified(null));
33+
assertThrows(IllegalArgumentException.class, () -> CommonUtil.makeQualified(""));
34+
35+
// Case 3: absolute path
36+
assertEquals("s3://bucket/cat", CommonUtil.makeQualified("s3://bucket/cat"));
37+
assertEquals("s3:///cat", CommonUtil.makeQualified("s3:///cat"));
38+
assertEquals("s3://bucket/cat", CommonUtil.makeQualified("s3://bucket/cat?param=foo"));
39+
assertEquals("s3://bucket/cat", CommonUtil.makeQualified("s3://bucket/cat#frag=0"));
40+
}
41+
}

‎java/lance-namespace-hive/src/main/java/com/lancedb/lance/namespace/hive/ErrorType.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.lancedb.lance.namespace.hive;
15+
16+
/** Error types for hive lance namespace */
17+
public enum ErrorType {
18+
HiveMetaStoreError("HiveMetaStoreError"),
19+
UnknownCatalog("UnknownCatalog"),
20+
CatalogAlreadyExist("CatalogAlreadyExist"),
21+
DatabaseAlreadyExist("DatabaseAlreadyExist");
22+
23+
private String type;
24+
25+
ErrorType(String type) {
26+
this.type = type;
27+
}
28+
29+
public String getType() {
30+
return type;
31+
}
32+
}

‎java/lance-namespace-hive/src/main/java/com/lancedb/lance/namespace/hive/Hive2Adapter.java‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.lancedb.lance.namespace.hive;
15+
16+
import com.lancedb.lance.namespace.LanceNamespaceException;
17+
import com.lancedb.lance.namespace.ObjectIdentifier;
18+
import com.lancedb.lance.namespace.model.CreateNamespaceRequest;
19+
import com.lancedb.lance.namespace.util.CommonUtil;
20+
import com.lancedb.lance.namespace.util.HiveUtil;
21+
import com.lancedb.lance.namespace.util.ValidationUtil;
22+
23+
import com.google.common.collect.Lists;
24+
import org.apache.hadoop.conf.Configuration;
25+
import org.apache.hadoop.hive.conf.HiveConf;
26+
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
27+
import org.apache.hadoop.hive.metastore.api.Database;
28+
import org.apache.thrift.TException;
29+
30+
import java.util.List;
31+
import java.util.Map;
32+
import java.util.function.Supplier;
33+
34+
import static com.lancedb.lance.namespace.hive.ErrorType.DatabaseAlreadyExist;
35+
import static com.lancedb.lance.namespace.hive.ErrorType.HiveMetaStoreError;
36+
37+
public class Hive2Adapter implements HiveAdapter {
38+
private HiveClientPool clientPool;
39+
private Configuration hadoopConf;
40+
41+
public Hive2Adapter(HiveClientPool clientPool, Configuration hadoopConf) {
42+
this.clientPool = clientPool;
43+
this.hadoopConf = hadoopConf;
44+
}
45+
46+
@Override
47+
public List<String> listNamespaces(ObjectIdentifier parent) {
48+
ValidationUtil.checkArgument(
49+
parent.levels() <= 2, "Expect a 2-level namespace but get %s", parent);
50+
51+
try {
52+
if (parent.isRoot()) {
53+
return clientPool.run(IMetaStoreClient::getAllDatabases);
54+
} else {
55+
return Lists.newArrayList();
56+
}
57+
} catch (TException | InterruptedException e) {
58+
if (e instanceof InterruptedException) {
59+
Thread.currentThread().interrupt();
60+
}
61+
throw LanceNamespaceException.serviceUnavailable(
62+
e.getMessage(), HiveMetaStoreError.getType(), "", CommonUtil.formatCurrentStackTrace());
63+
}
64+
}
65+
66+
@Override
67+
public void createNamespace(
68+
ObjectIdentifier id, CreateNamespaceRequest.ModeEnum mode, Map<String, String> properties) {
69+
ValidationUtil.checkArgument(id.levels() == 2, "Expect a 2-level namespace but get %s", id);
70+
71+
try {
72+
String db = id.level(0).toLowerCase();
73+
createDatabase(db, mode, properties);
74+
} catch (TException | InterruptedException e) {
75+
if (e instanceof InterruptedException) {
76+
Thread.currentThread().interrupt();
77+
}
78+
throw LanceNamespaceException.serviceUnavailable(
79+
e.getMessage(), HiveMetaStoreError.getType(), "", CommonUtil.formatCurrentStackTrace());
80+
}
81+
}
82+
83+
private void createDatabase(
84+
String dbName, CreateNamespaceRequest.ModeEnum mode, Map<String, String> properties)
85+
throws TException, InterruptedException {
86+
Database oldDb = HiveUtil.getDatabaseOrNull(clientPool, dbName);
87+
if (oldDb != null) {
88+
switch (mode) {
89+
case CREATE:
90+
throw LanceNamespaceException.conflict(
91+
String.format("Database %s already exist", dbName),
92+
DatabaseAlreadyExist.getType(),
93+
"",
94+
CommonUtil.formatCurrentStackTrace());
95+
case EXIST_OK:
96+
return;
97+
case OVERWRITE:
98+
clientPool.run(
99+
client -> {
100+
client.dropDatabase(dbName, false, true, false);
101+
return null;
102+
});
103+
}
104+
}
105+
106+
// Create database
107+
Supplier<String> warehouseLocation =
108+
() ->
109+
ValidationUtil.checkNotNullOrEmptyString(
110+
hadoopConf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname),
111+
String.format(
112+
"Warehouse location is not set: %s=null",
113+
HiveConf.ConfVars.METASTOREWAREHOUSE.varname));
114+
115+
Database database = new Database();
116+
database.setName(dbName);
117+
HiveUtil.setDatabaseProperties(database, warehouseLocation, dbName, properties);
118+
119+
clientPool.run(
120+
client -> {
121+
client.createDatabase(database);
122+
return null;
123+
});
124+
}
125+
}

0 commit comments

Comments
 (0)