Skip to content

Commit bf1951c

Browse files
authored
metrics: Enhance the o11y of TiFlash storage layer (release-7.5) (#10286)
close #10272 * Add metrics about * Number of keyspace, Storage instance, Segment instance, MemTable instance * Bytes of MemTable and allocated bytes of MemTable * System table * Add `delta_cache_alloc_size` to `system.dt_segments` * Add `column_count`,`delta_cache_alloc_size` to `system.dt_tables` Signed-off-by: JaySon-Huang <tshent@qq.com>
1 parent e8ea2e8 commit bf1951c

39 files changed

Lines changed: 6943 additions & 2161 deletions

CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ set(CMAKE_MACOSX_RPATH 1)
2424

2525
option(TIFLASH_ENABLE_LLVM_DEVELOPMENT "enable facilities for development with LLVM" OFF)
2626

27+
if(NOT CMAKE_PREFIX_PATH AND DEFINED ENV{CMAKE_PREFIX_PATH})
28+
message(STATUS "Reading CMAKE_PREFIX_PATH from env... $ENV{CMAKE_PREFIX_PATH}")
29+
set(CMAKE_PREFIX_PATH "$ENV{CMAKE_PREFIX_PATH}")
30+
endif()
2731
if(CMAKE_PREFIX_PATH)
2832
# append paths for cmake to check libs
2933
set(ENV{LD_LIBRARY_PATH}

dbms/src/Common/CurrentMetrics.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,13 @@
6060
M(DT_SnapshotOfPlaceIndex) \
6161
M(DT_SnapshotOfBitmapFilter) \
6262
M(DT_SnapshotOfDisaggReadNodeRead) \
63+
M(NumKeyspace) \
64+
M(NumIStorage) \
65+
M(DT_NumStorageDeltaMerge) \
66+
M(DT_NumSegment) \
67+
M(DT_NumMemTable) \
68+
M(DT_BytesMemTable) \
69+
M(DT_BytesMemTableAllocated) \
6370
M(IOLimiterPendingBgWriteReq) \
6471
M(IOLimiterPendingFgWriteReq) \
6572
M(IOLimiterPendingBgReadReq) \

dbms/src/Common/TiFlashMetrics.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,13 @@ namespace DB
347347
F(type_fg_write_alloc_bytes, {"type", "fg_write_alloc_bytes"}), \
348348
F(type_bg_write_req_bytes, {"type", "bg_write_req_bytes"}), \
349349
F(type_bg_write_alloc_bytes, {"type", "bg_write_alloc_bytes"})) \
350+
M(tiflash_storage_io_limiter_curr, \
351+
"Current limit bytes per second of Storage I/O limiter", \
352+
Gauge, \
353+
F(type_fg_read_bytes, {"type", "fg_read_bytes"}), \
354+
F(type_bg_read_bytes, {"type", "bg_read_bytes"}), \
355+
F(type_fg_write_bytes, {"type", "fg_write_bytes"}), \
356+
F(type_bg_write_bytes, {"type", "bg_write_bytes"})) \
350357
M(tiflash_storage_rough_set_filter_rate, \
351358
"Bucketed histogram of rough set filter rate", \
352359
Histogram, \

dbms/src/Encryption/RateLimiter.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,7 @@ void IORateLimiter::updateReadLimiter(Int64 bg_bytes, Int64 fg_bytes)
527527
{
528528
bg_read_limiter->updateMaxBytesPerSec(bg_bytes);
529529
}
530+
GET_METRIC(tiflash_storage_io_limiter_curr, type_bg_read_bytes).Set(bg_bytes);
530531

531532
if (fg_bytes == 0)
532533
{
@@ -540,6 +541,7 @@ void IORateLimiter::updateReadLimiter(Int64 bg_bytes, Int64 fg_bytes)
540541
{
541542
fg_read_limiter->updateMaxBytesPerSec(fg_bytes);
542543
}
544+
GET_METRIC(tiflash_storage_io_limiter_curr, type_fg_read_bytes).Set(fg_bytes);
543545
}
544546

545547
void IORateLimiter::updateWriteLimiter(Int64 bg_bytes, Int64 fg_bytes)
@@ -557,6 +559,7 @@ void IORateLimiter::updateWriteLimiter(Int64 bg_bytes, Int64 fg_bytes)
557559
{
558560
bg_write_limiter->updateMaxBytesPerSec(bg_bytes);
559561
}
562+
GET_METRIC(tiflash_storage_io_limiter_curr, type_bg_write_bytes).Set(bg_bytes);
560563

561564
if (fg_bytes == 0)
562565
{
@@ -570,6 +573,7 @@ void IORateLimiter::updateWriteLimiter(Int64 bg_bytes, Int64 fg_bytes)
570573
{
571574
fg_write_limiter->updateMaxBytesPerSec(fg_bytes);
572575
}
576+
GET_METRIC(tiflash_storage_io_limiter_curr, type_fg_write_bytes).Set(fg_bytes);
573577
}
574578

575579
void IORateLimiter::setBackgroundThreadIds(std::vector<pid_t> thread_ids)

dbms/src/Interpreters/AsynchronousMetrics.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,11 @@ void AsynchronousMetrics::update()
318318
{
319319
GET_METRIC(tiflash_storage_s3_gc_status, type_owner).Set(1.0);
320320
}
321+
else
322+
{
323+
// If the current node is not the owner, we reset the metric to 0
324+
GET_METRIC(tiflash_storage_s3_gc_status, type_owner).Set(0.0);
325+
}
321326
}
322327

323328
#if USE_MIMALLOC

dbms/src/Interpreters/InterpreterCreateQuery.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ class InterpreterCreateQuery : public IInterpreter
6868

6969
ASTPtr query_ptr;
7070
Context & context;
71-
std::string_view log_suffix;
71+
std::string log_suffix;
7272

7373
/// Using while loading database.
7474
ThreadPool * thread_pool = nullptr;

dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFile.h

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,9 @@ class ColumnFile
6565
: id(++MAX_COLUMN_FILE_ID)
6666
{}
6767

68+
public:
6869
virtual ~ColumnFile() = default;
6970

70-
public:
7171
enum Type : UInt32
7272
{
7373
DELETE_RANGE = 1,
@@ -96,8 +96,9 @@ class ColumnFile
9696
UInt64 getId() const { return id; }
9797

9898
virtual size_t getRows() const { return 0; }
99-
virtual size_t getBytes() const { return 0; };
100-
virtual size_t getDeletes() const { return 0; };
99+
virtual size_t getBytes() const { return 0; }
100+
virtual size_t getAllocateBytes() const { return 0; }
101+
virtual size_t getDeletes() const { return 0; }
101102

102103
virtual Type getType() const = 0;
103104

@@ -132,14 +133,19 @@ class ColumnFile
132133
virtual ColumnFileReaderPtr getReader(
133134
const DMContext & context,
134135
const IColumnFileDataProviderPtr & data_provider,
135-
const ColumnDefinesPtr & col_defs) const
136-
= 0;
136+
const ColumnDefinesPtr & col_defs) const = 0;
137137

138138
/// Note: Only ColumnFileInMemory can be appendable. Other ColumnFiles (i.e. ColumnFilePersisted) have
139139
/// been persisted in the disk and their data will be immutable.
140140
virtual bool isAppendable() const { return false; }
141141
virtual void disableAppend() {}
142-
virtual bool append(
142+
143+
struct AppendResult
144+
{
145+
bool success = false; // whether the append is successful
146+
size_t new_alloc_bytes = 0; // the new allocated bytes after append
147+
};
148+
virtual AppendResult append(
143149
const DMContext & /*dm_context*/,
144150
const Block & /*data*/,
145151
size_t /*offset*/,

dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.cpp

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ void ColumnFileInMemory::fillColumns(const ColumnDefines & col_defs, size_t col_
4242
// Copy data from cache
4343
const auto & type = getDataType(cd.id);
4444
auto col_data = type->createColumn();
45+
col_data->reserve(rows);
4546
col_data->insertRangeFrom(*(cache->block.getByPosition(col_offset).column), 0, rows);
4647
// Cast if need
4748
auto col_converted = convertColumnByColumnDefineIfNeed(type, std::move(col_data), cd);
@@ -64,36 +65,45 @@ ColumnFileReaderPtr ColumnFileInMemory::getReader(
6465
return std::make_shared<ColumnFileInMemoryReader>(*this, col_defs);
6566
}
6667

67-
bool ColumnFileInMemory::append(
68+
void ColumnFileInMemory::disableAppend()
69+
{
70+
disable_append = true;
71+
// TODO: Call shrinkToFit() to release the extra memory of the cache block.
72+
}
73+
74+
ColumnFile::AppendResult ColumnFileInMemory::append(
6875
const DMContext & context,
6976
const Block & data,
7077
size_t offset,
7178
size_t limit,
7279
size_t data_bytes)
7380
{
7481
if (disable_append)
75-
return false;
82+
return AppendResult{false, 0};
7683

7784
std::scoped_lock lock(cache->mutex);
7885
if (!isSameSchema(cache->block, data))
79-
return false;
86+
return AppendResult{false, 0};
8087

8188
// check whether this instance overflows
8289
if (cache->block.rows() >= context.delta_cache_limit_rows
8390
|| cache->block.bytes() >= context.delta_cache_limit_bytes)
84-
return false;
91+
return AppendResult{false, 0};
8592

93+
size_t new_alloc_block_bytes = 0;
8694
for (size_t i = 0; i < cache->block.columns(); ++i)
8795
{
8896
const auto & col = data.getByPosition(i).column;
8997
const auto & cache_col = *cache->block.getByPosition(i).column;
9098
auto * mutable_cache_col = const_cast<IColumn *>(&cache_col);
99+
size_t alloc_bytes = mutable_cache_col->allocatedBytes();
91100
mutable_cache_col->insertRangeFrom(*col, offset, limit);
101+
new_alloc_block_bytes += mutable_cache_col->allocatedBytes() - alloc_bytes;
92102
}
93103

94104
rows += limit;
95105
bytes += data_bytes;
96-
return true;
106+
return AppendResult{true, new_alloc_block_bytes};
97107
}
98108

99109
Block ColumnFileInMemory::readDataForFlush() const

dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileInMemory.h

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ class ColumnFileInMemory : public ColumnFile
6060
Type getType() const override { return Type::INMEMORY_FILE; }
6161

6262
size_t getRows() const override { return rows; }
63-
size_t getBytes() const override { return bytes; };
63+
size_t getBytes() const override { return bytes; }
64+
size_t getAllocateBytes() const override { return cache->block.allocatedBytes(); }
6465

6566
CachePtr getCache() { return cache; }
6667

@@ -75,9 +76,13 @@ class ColumnFileInMemory : public ColumnFile
7576
const ColumnDefinesPtr & col_defs) const override;
7677

7778
bool isAppendable() const override { return !disable_append; }
78-
void disableAppend() override { disable_append = true; }
79-
bool append(const DMContext & dm_context, const Block & data, size_t offset, size_t limit, size_t data_bytes)
80-
override;
79+
void disableAppend() override;
80+
AppendResult append(
81+
const DMContext & dm_context,
82+
const Block & data,
83+
size_t offset,
84+
size_t limit,
85+
size_t data_bytes) override;
8186

8287
Block readDataForFlush() const;
8388

dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,15 @@ size_t DeltaValueSpace::getTotalCacheBytes() const
247247
return mem_table_set->getBytes() + persisted_file_set->getTotalCacheBytes();
248248
}
249249

250+
size_t DeltaValueSpace::getTotalAllocatedBytes() const
251+
{
252+
std::scoped_lock lock(mutex);
253+
return mem_table_set->getAllocatedBytes();
254+
}
255+
250256
size_t DeltaValueSpace::getValidCacheRows() const
251257
{
258+
// FIXME: Seems that this function is the same as getTotalCacheRows().
252259
std::scoped_lock lock(mutex);
253260
return mem_table_set->getRows() + persisted_file_set->getValidCacheRows();
254261
}

0 commit comments

Comments
 (0)