HIVE-29694

ql/src/test/queries/clientpositive/partition_descriptor_reuse.q

@@ -0,0 +1,19 @@
+-- Debug instructions:
+-- 1. Run the test in debug mode: mvn test -Dtest=TestMiniLlapLocalCliDriver -Dqfile=partition_descriptor_reuse.q -Dtest.output.overwrite -Dtest.metastore.db=postgres -Dmaven.surefire.debug
+-- 2. Put a breakpoint in CliAdapter.java:120 CliAdapter.this.tearDown() to prevent DB from being dropped.
+-- 3. Connect to the dockerized database and check the state of the metadata
+-- docker exec -it CONTAINER_NAME psql -U test -d hivedb
+-- TODO: Probably not gonna need this file at the end so drop it
+
+set metastore.add.partition.reuse.existing.column.descriptors=true;
+
+create table person(id string, fname string) partitioned by (birthyear string);
+alter table person add partition (birthyear='1987');
+alter table person add partition (birthyear='1988');
+alter table person add partition (birthyear='1989');
+alter table person add partition (birthyear='1990');
+alter table person add partition (birthyear='1991');
+
+alter table person partition (birthyear='1989') add columns (lname string);
+alter table person partition (birthyear='1990') add columns (lname string);
+alter table person partition (birthyear='1991') change column fname fullname string;

ql/src/test/results/clientpositive/llap/partition_descriptor_reuse.q.out

@@ -0,0 +1,70 @@
+PREHOOK: query: create table person(id string, fname string) partitioned by (birthyear string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@person
+POSTHOOK: query: create table person(id string, fname string) partitioned by (birthyear string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@person
+PREHOOK: query: alter table person add partition (birthyear='1987')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@person
+POSTHOOK: query: alter table person add partition (birthyear='1987')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@person
+POSTHOOK: Output: default@person@birthyear=1987
+PREHOOK: query: alter table person add partition (birthyear='1988')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@person
+POSTHOOK: query: alter table person add partition (birthyear='1988')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@person
+POSTHOOK: Output: default@person@birthyear=1988
+PREHOOK: query: alter table person add partition (birthyear='1989')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@person
+POSTHOOK: query: alter table person add partition (birthyear='1989')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@person
+POSTHOOK: Output: default@person@birthyear=1989
+PREHOOK: query: alter table person add partition (birthyear='1990')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@person
+POSTHOOK: query: alter table person add partition (birthyear='1990')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@person
+POSTHOOK: Output: default@person@birthyear=1990
+PREHOOK: query: alter table person add partition (birthyear='1991')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@person
+POSTHOOK: query: alter table person add partition (birthyear='1991')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@person
+POSTHOOK: Output: default@person@birthyear=1991
+PREHOOK: query: alter table person partition (birthyear='1989') add columns (lname string)
+PREHOOK: type: ALTERTABLE_ADDCOLS
+PREHOOK: Input: default@person
+PREHOOK: Output: default@person@birthyear=1989
+POSTHOOK: query: alter table person partition (birthyear='1989') add columns (lname string)
+POSTHOOK: type: ALTERTABLE_ADDCOLS
+POSTHOOK: Input: default@person
+POSTHOOK: Input: default@person@birthyear=1989
+POSTHOOK: Output: default@person@birthyear=1989
+PREHOOK: query: alter table person partition (birthyear='1990') add columns (lname string)
+PREHOOK: type: ALTERTABLE_ADDCOLS
+PREHOOK: Input: default@person
+PREHOOK: Output: default@person@birthyear=1990
+POSTHOOK: query: alter table person partition (birthyear='1990') add columns (lname string)
+POSTHOOK: type: ALTERTABLE_ADDCOLS
+POSTHOOK: Input: default@person
+POSTHOOK: Input: default@person@birthyear=1990
+POSTHOOK: Output: default@person@birthyear=1990
+PREHOOK: query: alter table person partition (birthyear='1991') change column fname fullname string
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@person
+PREHOOK: Output: default@person@birthyear=1991
+POSTHOOK: query: alter table person partition (birthyear='1991') change column fname fullname string
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@person
+POSTHOOK: Input: default@person@birthyear=1991
+POSTHOOK: Output: default@person@birthyear=1991

standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java

@@ -17,6 +17,16 @@
  */
 package org.apache.hadoop.hive.metastore.conf;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.ZooKeeperHiveHelper;
+import org.apache.hadoop.hive.metastore.utils.StringUtils;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hive.common.util.SuppressFBWarnings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
@@ -33,17 +43,6 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.ZooKeeperHiveHelper;
-import org.apache.hadoop.hive.metastore.utils.StringUtils;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hive.common.util.SuppressFBWarnings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-
 /**
  * A set of definitions of config values used by the Metastore.  One of the key aims of this
  * class is to provide backwards compatibility with existing Hive configuration keys while
@@ -312,12 +311,12 @@ public enum ConfVars {
     ACID_HOUSEKEEPER_SERVICE_INTERVAL("metastore.acid.housekeeper.interval",
         "hive.metastore.acid.housekeeper.interval", 60, TimeUnit.SECONDS,
         "Time interval describing how often the acid housekeeper runs."),
-    COMPACTION_HOUSEKEEPER_SERVICE_INTERVAL("metastore.compaction.housekeeper.interval",
-        "hive.metastore.compaction.housekeeper.interval", 300, TimeUnit.SECONDS,
-        "Time interval describing how often the acid compaction housekeeper runs."),
     ACID_TXN_CLEANER_INTERVAL("metastore.acid.txn.cleaner.interval",
         "hive.metastore.acid.txn.cleaner.interval", 10, TimeUnit.SECONDS,
         "Time interval describing how often aborted and committed txns are cleaned."),
+    ADD_PARTITION_REUSE_EXISTING_COLUMN_DESCRIPTORS("metastore.add.partition.reuse.existing.column.descriptors",
+        "hive.metastore.add.partition.reuse.existing.column.descriptors", false,
+        "Add partition reuse existing column descriptors to avoid metadata bloat on schema evolution."),
     ADDED_JARS("metastore.added.jars.path", "hive.added.jars.path", "",
         "This an internal parameter."),
     AGGREGATE_STATS_CACHE_CLEAN_UNTIL("metastore.aggregate.stats.cache.clean.until",
@@ -432,6 +431,9 @@ public enum ConfVars {
             TimeUnit.SECONDS, "MetaStore Client socket timeout in seconds"),
     CLIENT_CONNECTION_TIMEOUT("metastore.client.connection.timeout", "hive.metastore.client.connection.timeout", 600,
             TimeUnit.SECONDS, "MetaStore Client connection timeout in seconds"),
+    COMPACTION_HOUSEKEEPER_SERVICE_INTERVAL("metastore.compaction.housekeeper.interval",
+        "hive.metastore.compaction.housekeeper.interval", 300, TimeUnit.SECONDS,
+        "Time interval describing how often the acid compaction housekeeper runs."),
     COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE("metastore.compactor.history.retention.did.not.initiate",
         "hive.compactor.history.retention.did.not.initiate", 2,
         new RangeValidator(0, 100), "Determines how many compaction records in state " +

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ColumnDescriptorSupplier.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.model.MColumnDescriptor;
+import org.apache.hadoop.hive.metastore.model.MTable;
+
+import java.util.List;
+
+public interface ColumnDescriptorSupplier {
+  /**
+   * Gets a column descriptor for the provided table and list of columns. If no matching column descriptor exists,
+   * returns null. The column descriptor is considered a match if it has the same number of columns, and each column has
+   * the same name, type, and comment in the same order.
+   *
+   * @param cols the columns to match
+   * @param mt the table to search
+   * @return an existing MColumnDescriptor if found, null otherwise
+   */
+  MColumnDescriptor getColumnDescriptor(List<FieldSchema> cols, MTable mt) throws MetaException;
+}