Merge pull request #465 from iskakaushik/cache-type-codes-perf

Cache type codes to eliminate shared_ptr overhead in hot paths

Author: slabko

clickhouse/columns/decimal.cpp

@@ -109,21 +109,27 @@ ColumnDecimal::ColumnDecimal(size_t precision, size_t scale)
     } else {
         data_ = std::make_shared<ColumnInt128>();
     }
+    data_type_code_ = data_->Type()->GetCode();
 }
 
 ColumnDecimal::ColumnDecimal(TypeRef type, ColumnRef data)
     : Column(type),
-      data_(data)
+      data_(data),
+      data_type_code_(data_->Type()->GetCode())
 {
 }
 
 void ColumnDecimal::Append(const Int128& value) {
-    if (data_->Type()->GetCode() == Type::Int32) {
-        data_->As<ColumnInt32>()->Append(static_cast<ColumnInt32::DataType>(value));
-    } else if (data_->Type()->GetCode() == Type::Int64) {
-        data_->As<ColumnInt64>()->Append(static_cast<ColumnInt64::DataType>(value));
-    } else {
-        data_->As<ColumnInt128>()->Append(static_cast<ColumnInt128::DataType>(value));
+    switch (data_type_code_) {
+        case Type::Int32:
+            static_cast<ColumnInt32*>(data_.get())->Append(static_cast<int32_t>(value));
+            break;
+        case Type::Int64:
+            static_cast<ColumnInt64*>(data_.get())->Append(static_cast<int64_t>(value));
+            break;
+        default:
+            static_cast<ColumnInt128*>(data_.get())->Append(static_cast<Int128>(value));
+            break;
     }
 }
 
@@ -179,13 +185,13 @@ void ColumnDecimal::Append(const std::string& value) {
 }
 
 Int128 ColumnDecimal::At(size_t i) const {
-    switch (data_->Type()->GetCode()) {
+    switch (data_type_code_) {
         case Type::Int32:
-            return static_cast<Int128>(data_->As<ColumnInt32>()->At(i));
+            return static_cast<Int128>(static_cast<const ColumnInt32*>(data_.get())->At(i));
         case Type::Int64:
-            return static_cast<Int128>(data_->As<ColumnInt64>()->At(i));
+            return static_cast<Int128>(static_cast<const ColumnInt64*>(data_.get())->At(i));
         case Type::Int128:
-            return data_->As<ColumnInt128>()->At(i);
+            return static_cast<const ColumnInt128*>(data_.get())->At(i);
         default:
             throw ValidationError("Invalid data_ column type in ColumnDecimal");
     }

clickhouse/columns/decimal.h

@@ -42,6 +42,7 @@ class ColumnDecimal : public Column {
     ///  - ColumnInt64
     ///  - ColumnInt128
     ColumnRef data_;
+    Type::Code data_type_code_;
 
     explicit ColumnDecimal(TypeRef type, ColumnRef data);
 };

clickhouse/columns/lowcardinality.cpp

@@ -158,15 +158,17 @@ namespace clickhouse {
 ColumnLowCardinality::ColumnLowCardinality(ColumnRef dictionary_column)
     : Column(Type::CreateLowCardinality(dictionary_column->Type())),
       dictionary_column_(dictionary_column->CloneEmpty()), // safe way to get an column of the same type.
-      index_column_(std::make_shared<ColumnUInt32>())
+      index_column_(std::make_shared<ColumnUInt32>()),
+      index_type_code_(Type::UInt32)
 {
     Setup(dictionary_column);
 }
 
 ColumnLowCardinality::ColumnLowCardinality(std::shared_ptr<ColumnNullable> dictionary_column)
     : Column(Type::CreateLowCardinality(dictionary_column->Type())),
       dictionary_column_(dictionary_column->CloneEmpty()), // safe way to get an column of the same type.
-      index_column_(std::make_shared<ColumnUInt32>())
+      index_column_(std::make_shared<ColumnUInt32>()),
+      index_type_code_(Type::UInt32)
 {
     AppendNullItem();
     Setup(dictionary_column);
@@ -200,22 +202,65 @@ void ColumnLowCardinality::Setup(ColumnRef dictionary_column) {
 }
 
 std::uint64_t ColumnLowCardinality::getDictionaryIndex(std::uint64_t item_index) const {
-    return VisitIndexColumn([item_index](const auto & arg) -> std::uint64_t {
-        return arg[item_index];
-    }, *index_column_);
+    switch (index_type_code_) {
+        case Type::UInt8:
+            return static_cast<const ColumnUInt8&>(*index_column_)[item_index];
+        case Type::UInt16:
+            return static_cast<const ColumnUInt16&>(*index_column_)[item_index];
+        case Type::UInt32:
+            return static_cast<const ColumnUInt32&>(*index_column_)[item_index];
+        case Type::UInt64:
+            return static_cast<const ColumnUInt64&>(*index_column_)[item_index];
+        default:
+            throw ValidationError("Invalid index column type");
+    }
 }
 
 void ColumnLowCardinality::appendIndex(std::uint64_t item_index) {
     // TODO (nemkov): handle case when index should go from UInt8 to UInt16, etc.
-    VisitIndexColumn([item_index](auto & arg) {
-        arg.Append(static_cast<typename std::decay_t<decltype(arg)>::DataType>(item_index));
-    }, *index_column_);
+    switch (index_type_code_) {
+        case Type::UInt8:
+            static_cast<ColumnUInt8&>(*index_column_).Append(static_cast<uint8_t>(item_index));
+            break;
+        case Type::UInt16:
+            static_cast<ColumnUInt16&>(*index_column_).Append(static_cast<uint16_t>(item_index));
+            break;
+        case Type::UInt32:
+            static_cast<ColumnUInt32&>(*index_column_).Append(static_cast<uint32_t>(item_index));
+            break;
+        case Type::UInt64:
+            static_cast<ColumnUInt64&>(*index_column_).Append(static_cast<uint64_t>(item_index));
+            break;
+        default:
+            throw ValidationError("Invalid index column type");
+    }
 }
 
 void ColumnLowCardinality::removeLastIndex() {
-    VisitIndexColumn([](auto & arg) {
-        arg.Erase(arg.Size() - 1);
-    }, *index_column_);
+    switch (index_type_code_) {
+        case Type::UInt8: {
+            auto& col = static_cast<ColumnUInt8&>(*index_column_);
+            col.Erase(col.Size() - 1);
+            break;
+        }
+        case Type::UInt16: {
+            auto& col = static_cast<ColumnUInt16&>(*index_column_);
+            col.Erase(col.Size() - 1);
+            break;
+        }
+        case Type::UInt32: {
+            auto& col = static_cast<ColumnUInt32&>(*index_column_);
+            col.Erase(col.Size() - 1);
+            break;
+        }
+        case Type::UInt64: {
+            auto& col = static_cast<ColumnUInt64&>(*index_column_);
+            col.Erase(col.Size() - 1);
+            break;
+        }
+        default:
+            throw ValidationError("Invalid index column type");
+    }
 }
 
 details::LowCardinalityHashKey ColumnLowCardinality::computeHashKey(const ItemView & item) {
@@ -337,6 +382,7 @@ bool ColumnLowCardinality::LoadBody(InputStream* input, size_t rows) {
         dictionary_column_->Swap(*new_dictionary);
         index_column_.swap(new_index);
         unique_items_map_.swap(new_unique_items_map);
+        index_type_code_ = index_column_->Type()->GetCode();
 
         return true;
     } catch (...) {
@@ -411,6 +457,7 @@ void ColumnLowCardinality::Swap(Column& other) {
 
     index_column_.swap(col.index_column_);
     unique_items_map_.swap(col.unique_items_map_);
+    std::swap(index_type_code_, col.index_type_code_);
 }
 
 ItemView ColumnLowCardinality::GetItem(size_t index) const {

clickhouse/columns/lowcardinality.h

@@ -110,6 +110,8 @@ class ColumnLowCardinality : public Column {
     void AppendNullItem();
     void AppendDefaultItem();
 
+    Type::Code index_type_code_;
+
 public:
     static details::LowCardinalityHashKey computeHashKey(const ItemView &);
 };