Skip to content

Commit 4991c8d

Browse files
zhangbutaozhangbutao
committed
Distinguish the location of the database by catalog
1 parent 6484ad3 commit 4991c8d

23 files changed

Lines changed: 319 additions & 98 deletions

File tree

‎common/src/java/org/apache/hadoop/hive/conf/HiveConf.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -922,6 +922,15 @@ public static enum ConfVars {
922922
"Default location for external tables created in the warehouse. " +
923923
"If not set or null, then the normal warehouse location will be used as the default location."),
924924

925+
METASTORE_CATALOG_WAREHOUSE("hive.metastore.warehouse.catalog.dir", "/user/hive/catalog/warehouse",
926+
"location of default database for the warehouse. If the name of the catalog to which the db belongs is testcat," +
927+
"then the default path prefix for the db is /user/hive/catalog/warehouse/testcat"),
928+
929+
HIVE_METASTORE_CATALOG_WAREHOUSE_EXTERNAL("hive.metastore.warehouse.catalog.external.dir", null,
930+
"Default location for external tables created in the warehouse. " +
931+
"If not set or null, then the normal warehouse location(MetastoreConf.METASTORE_CATALOG_WAREHOUSE)" +
932+
"will be used as the default location."),
933+
925934
/**
926935
* @deprecated Use MetastoreConf.THRIFT_URIS
927936
*/

‎itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -544,6 +544,7 @@ private void setFsRelatedProperties(HiveConf conf, boolean isLocalFs, FileSystem
544544
// Different paths if running locally vs a remote fileSystem. Ideally this difference should not
545545
// exist.
546546
Path warehousePath;
547+
Path warehouseCatPath;
547548
Path jarPath;
548549
Path userInstallPath;
549550
if (isLocalFs) {
@@ -554,23 +555,27 @@ private void setFsRelatedProperties(HiveConf conf, boolean isLocalFs, FileSystem
554555
// Create a fake fs root for local fs
555556
Path localFsRoot = new Path(path, "localfs");
556557
warehousePath = new Path(localFsRoot, "warehouse");
558+
warehouseCatPath = new Path(localFsRoot, "catalog");
557559
jarPath = new Path(localFsRoot, "jar");
558560
userInstallPath = new Path(localFsRoot, "user_install");
559561
} else {
560562
// TODO Why is this changed from the default in hive-conf?
561563
warehousePath = new Path(fsUriString, "/build/ql/test/data/warehouse/");
564+
warehouseCatPath = new Path(fsUriString, "/build/ql/test/data/catalog/");
562565
jarPath = new Path(new Path(fsUriString, "/user"), "hive");
563566
userInstallPath = new Path(fsUriString, "/user");
564567
}
565568

566569
warehousePath = fs.makeQualified(warehousePath);
570+
warehouseCatPath = fs.makeQualified(warehouseCatPath);
567571
jarPath = fs.makeQualified(jarPath);
568572
userInstallPath = fs.makeQualified(userInstallPath);
569573

570574
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsUriString);
571575

572576
// Remote dirs
573577
conf.setVar(ConfVars.METASTORE_WAREHOUSE, warehousePath.toString());
578+
conf.setVar(ConfVars.METASTORE_CATALOG_WAREHOUSE, warehouseCatPath.toString());
574579
conf.setVar(ConfVars.HIVE_JAR_DIRECTORY, jarPath.toString());
575580
conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, userInstallPath.toString());
576581
// ConfVars.SCRATCH_DIR - {test.tmp.dir}/scratchdir

‎parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1131,10 +1131,10 @@ createCatalogStatement
11311131
: KW_CREATE KW_CATALOG
11321132
ifNotExists?
11331133
name=identifier
1134-
catLocation
1134+
catLocation?
11351135
catalogComment?
1136-
(KW_PROPERTIES catprops=properties)?
1137-
-> ^(TOK_CREATECATALOG $name catLocation ifNotExists? catalogComment? $catprops?)
1136+
KW_PROPERTIES catprops=properties
1137+
-> ^(TOK_CREATECATALOG $name catLocation? ifNotExists? catalogComment? $catprops)
11381138
;
11391139
11401140
catLocation

‎ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,11 @@
2929
import org.apache.hadoop.hive.common.TableName;
3030
import org.apache.hadoop.hive.conf.HiveConf;
3131
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
32+
import org.apache.hadoop.hive.metastore.CatalogUtil;
3233
import org.apache.hadoop.hive.metastore.HiveMetaHook;
3334
import org.apache.hadoop.hive.metastore.TableType;
35+
import org.apache.hadoop.hive.metastore.Warehouse;
36+
import org.apache.hadoop.hive.metastore.api.Catalog;
3437
import org.apache.hadoop.hive.metastore.api.Database;
3538
import org.apache.hadoop.hive.metastore.api.FieldSchema;
3639
import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
@@ -292,4 +295,11 @@ public static Pair<String, String> getCatDbNamePair(ASTNode dbNameNode) throws S
292295

293296
return Pair.of(catName, dbName);
294297
}
298+
299+
public static boolean doesCatalogSupportCreateDB(Catalog catalog) {
300+
if (catalog.getName().equals(Warehouse.DEFAULT_CATALOG_NAME)) {
301+
return true;
302+
}
303+
return CatalogUtil.CatalogType.NATIVE.toString().toLowerCase().equals(catalog.getParameters().get("type"));
304+
}
295305
}

‎ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/create/CreateCatalogAnalyzer.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
package org.apache.hadoop.hive.ql.ddl.catalog.create;
2020

21+
import com.google.common.base.Strings;
22+
import org.apache.hadoop.hive.metastore.CatalogUtil;
2123
import org.apache.hadoop.hive.metastore.api.Catalog;
2224
import org.apache.hadoop.hive.ql.QueryState;
2325
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory;
@@ -43,16 +45,19 @@ public CreateCatalogAnalyzer(QueryState queryState) throws SemanticException {
4345
@Override
4446
public void analyzeInternal(ASTNode root) throws SemanticException {
4547
String catalogName = unescapeIdentifier(root.getChild(0).getText());
46-
String locationUrl = unescapeSQLString(root.getChild(1).getChild(0).getText());
47-
outputs.add(toWriteEntity(locationUrl));
4848

49+
String locationUrl = null;
4950
boolean ifNotExists = false;
5051
String comment = null;
5152
Map<String, String> props = null;
5253

53-
for (int i = 2; i < root.getChildCount(); i++) {
54+
for (int i = 1; i < root.getChildCount(); i++) {
5455
ASTNode childNode = (ASTNode) root.getChild(i);
5556
switch (childNode.getToken().getType()) {
57+
case HiveParser.TOK_CATALOGLOCATION:
58+
locationUrl = unescapeSQLString(childNode.getChild(0).getText());
59+
outputs.add(toWriteEntity(locationUrl));
60+
break;
5661
case HiveParser.TOK_IFNOTEXISTS:
5762
ifNotExists = true;
5863
break;
@@ -67,10 +72,23 @@ public void analyzeInternal(ASTNode root) throws SemanticException {
6772
}
6873
}
6974

75+
assert props != null;
76+
checkCatalogType(props);
77+
7078
CreateCatalogDesc desc = new CreateCatalogDesc(catalogName, comment, locationUrl, ifNotExists, props);
7179
Catalog catalog = new Catalog(catalogName, locationUrl);
7280

7381
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)));
7482
outputs.add(new WriteEntity(catalog, WriteEntity.WriteType.DDL_NO_LOCK));
7583
}
84+
85+
private static void checkCatalogType(Map<String, String> props) throws SemanticException {
86+
String catalogType = props.get("type");
87+
if (Strings.isNullOrEmpty(catalogType)) {
88+
throw new SemanticException("'type' can not be null or empty");
89+
}
90+
if (!CatalogUtil.isValidCatalogType(catalogType)) {
91+
throw new SemanticException(String.format("type '%s' is not valid", catalogType));
92+
}
93+
}
7694
}

‎ql/src/java/org/apache/hadoop/hive/ql/ddl/catalog/create/CreateCatalogOperation.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
2222
import org.apache.hadoop.hive.metastore.api.Catalog;
23+
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
2324
import org.apache.hadoop.hive.ql.ErrorMsg;
2425
import org.apache.hadoop.hive.ql.ddl.DDLOperation;
2526
import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
@@ -37,7 +38,11 @@ public CreateCatalogOperation(DDLOperationContext context, CreateCatalogDesc des
3738

3839
@Override
3940
public int execute() throws Exception {
40-
Catalog catalog = new Catalog(desc.getName(), desc.getLocationUri());
41+
String catLocationUri = Optional.ofNullable(desc.getLocationUri())
42+
.orElse(MetastoreConf.getVar(context.getConf(),
43+
MetastoreConf.ConfVars.WAREHOUSE_CATALOG) + "/" + desc.getName());
44+
45+
Catalog catalog = new Catalog(desc.getName(), catLocationUri);
4146
catalog.setDescription(desc.getComment());
4247
Optional.ofNullable(desc.getCatlogProperties())
4348
.ifPresent(catalog::setParameters);

‎ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseAnalyzer.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@
1919
package org.apache.hadoop.hive.ql.ddl.database.create;
2020

2121
import java.util.Map;
22+
import java.util.Optional;
2223

2324
import org.apache.commons.lang3.tuple.Pair;
25+
import org.apache.hadoop.hive.metastore.api.Catalog;
2426
import org.apache.hadoop.hive.metastore.api.DataConnector;
2527
import org.apache.hadoop.hive.metastore.api.Database;
2628
import org.apache.hadoop.hive.metastore.api.DatabaseType;
@@ -33,6 +35,7 @@
3335
import org.apache.hadoop.hive.ql.ddl.DDLWork;
3436
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
3537
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
38+
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
3639
import org.apache.hadoop.hive.ql.parse.ASTNode;
3740
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
3841
import org.apache.hadoop.hive.ql.parse.HiveParser;
@@ -51,10 +54,16 @@ public CreateDatabaseAnalyzer(QueryState queryState) throws SemanticException {
5154
@Override
5255
public void analyzeInternal(ASTNode root) throws SemanticException {
5356
Pair<String, String> catDbNamePair = DDLUtils.getCatDbNamePair((ASTNode) root.getChild(0));
54-
String catalogName = catDbNamePair.getLeft();
55-
if (catalogName != null && getCatalog(catalogName) == null) {
57+
String catalogName = Optional.ofNullable(catDbNamePair.getLeft())
58+
.orElse(HiveUtils.getCurrentCatalogOrDefault(conf));
59+
60+
Catalog catalog = getCatalog(catalogName);
61+
if (catalog == null) {
5662
throw new SemanticException(ErrorMsg.CATALOG_NOT_EXISTS, catalogName);
5763
}
64+
if (!DDLUtils.doesCatalogSupportCreateDB(catalog)) {
65+
throw new SemanticException("Catalog %s does not support creating database".formatted(catalogName));
66+
}
5867
String databaseName = catDbNamePair.getRight();
5968

6069
boolean ifNotExists = false;

‎ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseOperation.java‎

Lines changed: 45 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,12 @@ public int execute() throws HiveException {
5656
if (desc.getManagedLocationUri() != null) {
5757
database.setManagedLocationUri(desc.getManagedLocationUri());
5858
}
59-
makeLocationQualified(database); // TODO catalog. Add catalog prefix for db location. Depend on HIVE-29241.
59+
makeLocationQualified(database);
6060
if (database.getLocationUri().equalsIgnoreCase(database.getManagedLocationUri())) {
6161
throw new HiveException("Managed and external locations for database cannot be the same");
6262
}
6363
} else if (desc.getDatabaseType() == DatabaseType.REMOTE) {
64-
makeLocationQualified(database); // TODO catalog. Add catalog prefix for db location. Depend on HIVE-29241.
64+
makeLocationQualified(database);
6565
database.setConnector_name(desc.getConnectorName());
6666
database.setRemote_dbname(desc.getRemoteDbName());
6767
} else { // should never be here
@@ -81,34 +81,62 @@ public int execute() throws HiveException {
8181
}
8282

8383
private void makeLocationQualified(Database database) throws HiveException {
84+
String catalogName = database.getCatalogName().toLowerCase();
85+
String dbName = database.getName().toLowerCase();
86+
boolean isDefaultCatalog = Warehouse.DEFAULT_CATALOG_NAME.equalsIgnoreCase(catalogName);
87+
88+
// -------- External location --------
8489
if (database.isSetLocationUri()) {
8590
database.setLocationUri(Utilities.getQualifiedPath(context.getConf(), new Path(database.getLocationUri())));
8691
} else {
87-
// Location is not set we utilize WAREHOUSE_EXTERNAL together with database name
88-
String rootDir = MetastoreConf.getVar(context.getConf(), MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL);
89-
if (rootDir == null || rootDir.trim().isEmpty()) {
90-
// Fallback plan
91-
LOG.warn(String.format(
92-
"%s is not set, falling back to %s. This could cause external tables to use to managed tablespace.",
93-
MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname(), MetastoreConf.ConfVars.WAREHOUSE.getVarname()));
94-
rootDir = MetastoreConf.getVar(context.getConf(), MetastoreConf.ConfVars.WAREHOUSE);
95-
}
96-
Path path = new Path(rootDir, database.getName().toLowerCase() + DATABASE_PATH_SUFFIX);
97-
String qualifiedPath = Utilities.getQualifiedPath(context.getConf(), path);
98-
database.setLocationUri(qualifiedPath);
92+
String rootDir = getExternalRootDir(isDefaultCatalog);
93+
Path path = buildDbPath(rootDir, catalogName, dbName, isDefaultCatalog);
94+
database.setLocationUri(Utilities.getQualifiedPath(context.getConf(), path));
9995
}
10096

97+
// -------- Managed location --------
10198
if (database.isSetManagedLocationUri()) {
10299
database.setManagedLocationUri(Utilities.getQualifiedPath(context.getConf(),
103100
new Path(database.getManagedLocationUri())));
104101
} else {
105-
// ManagedLocation is not set we utilize WAREHOUSE together with database name
106-
String rootDir = MetastoreConf.getVar(context.getConf(), MetastoreConf.ConfVars.WAREHOUSE);
107-
Path path = new Path(rootDir, database.getName().toLowerCase() + DATABASE_PATH_SUFFIX);
102+
String rootDir = MetastoreConf.getVar(
103+
context.getConf(),
104+
isDefaultCatalog
105+
? MetastoreConf.ConfVars.WAREHOUSE
106+
: MetastoreConf.ConfVars.WAREHOUSE_CATALOG
107+
);
108+
109+
Path path = buildDbPath(rootDir, catalogName, dbName, isDefaultCatalog);
108110
String qualifiedPath = Utilities.getQualifiedPath(context.getConf(), path);
109111
if (!qualifiedPath.equals(database.getLocationUri())) {
110112
database.setManagedLocationUri(qualifiedPath);
111113
}
112114
}
113115
}
116+
117+
private Path buildDbPath(String rootDir, String catalogName, String dbName, boolean isDefaultCatalog) {
118+
return isDefaultCatalog
119+
? new Path(rootDir, dbName + DATABASE_PATH_SUFFIX)
120+
: new Path(rootDir + "/" + catalogName, dbName + DATABASE_PATH_SUFFIX);
121+
}
122+
123+
private String getExternalRootDir(boolean isDefaultCatalog) {
124+
MetastoreConf.ConfVars externalVar = isDefaultCatalog
125+
? MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL
126+
: MetastoreConf.ConfVars.WAREHOUSE_CATALOG_EXTERNAL;
127+
128+
String rootDir = MetastoreConf.getVar(context.getConf(), externalVar);
129+
if (rootDir != null && !rootDir.trim().isEmpty()) {
130+
return rootDir;
131+
}
132+
133+
MetastoreConf.ConfVars fallbackVar = isDefaultCatalog
134+
? MetastoreConf.ConfVars.WAREHOUSE
135+
: MetastoreConf.ConfVars.WAREHOUSE_CATALOG;
136+
137+
LOG.warn("{} is not set, falling back to {}. This could cause external tables to use managed tablespace.",
138+
externalVar.getVarname(), fallbackVar.getVarname());
139+
140+
return MetastoreConf.getVar(context.getConf(), fallbackVar);
141+
}
114142
}

‎ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.hadoop.fs.PathFilter;
2828
import org.apache.hadoop.hive.common.repl.ReplConst;
2929
import org.apache.hadoop.hive.conf.HiveConf;
30+
import org.apache.hadoop.hive.metastore.Warehouse;
3031
import org.apache.hadoop.hive.metastore.api.Database;
3132
import org.apache.hadoop.hive.metastore.api.FieldSchema;
3233
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
@@ -388,9 +389,15 @@ public static void createDbExportDump(FileSystem fs, Path metadataPath, Database
388389

389390
private static void updateIfCustomDbLocations(Database database, Configuration conf) throws SemanticException {
390391
try {
391-
String whLocatoion = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL,
392-
MetastoreConf.getVar(conf, MetastoreConf.ConfVars.WAREHOUSE));
393-
Path dbDerivedLoc = new Path(whLocatoion, database.getName().toLowerCase() + DATABASE_PATH_SUFFIX);
392+
boolean isDefaultCatalog = Warehouse.DEFAULT_CATALOG_NAME.equals(database.getCatalogName());
393+
String whLocation = MetastoreConf.getVar(conf,
394+
isDefaultCatalog ? MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL : MetastoreConf.ConfVars.WAREHOUSE_CATALOG_EXTERNAL,
395+
MetastoreConf.getVar(conf,
396+
isDefaultCatalog ? MetastoreConf.ConfVars.WAREHOUSE : MetastoreConf.ConfVars.WAREHOUSE_CATALOG
397+
)
398+
);
399+
400+
Path dbDerivedLoc = new Path(whLocation, database.getName().toLowerCase() + DATABASE_PATH_SUFFIX);
394401
String defaultDbLoc = Utilities.getQualifiedPath((HiveConf) conf, dbDerivedLoc);
395402
database.putToParameters(ReplConst.REPL_IS_CUSTOM_DB_LOC,
396403
Boolean.toString(!defaultDbLoc.equals(database.getLocationUri())));

‎ql/src/java/org/apache/hadoop/hive/ql/queryhistory/repository/AbstractRepository.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ protected Database initDatabase(Hive hive) {
8282
db = hive.getDatabase(QUERY_HISTORY_DB_NAME);
8383
if (db == null) {
8484
LOG.warn("Database ({}) for query history table hasn't been found, auto-creating one", QUERY_HISTORY_DB_NAME);
85-
String location = getDatabaseLocation(QUERY_HISTORY_DB_NAME);
85+
String location = getDatabaseLocation(db);
8686
db = new Database(QUERY_HISTORY_DB_NAME, QUERY_HISTORY_DB_COMMENT,
8787
location, null);
8888
hive.createDatabase(db, false);
@@ -93,8 +93,8 @@ protected Database initDatabase(Hive hive) {
9393
}
9494
}
9595

96-
private String getDatabaseLocation(String databaseName) throws Exception {
97-
return warehouse.getDefaultExternalDatabasePath(databaseName).toUri().toString();
96+
private String getDatabaseLocation(Database db) throws Exception {
97+
return warehouse.getDefaultExternalDatabasePath(db).toUri().toString();
9898
}
9999

100100
protected Table initTable(Hive hive, Database db) {

0 commit comments

Comments
 (0)