From 2cd145c5e3f04251b61d40bb40a9c68598e8b382 Mon Sep 17 00:00:00 2001 From: Ray Yan Date: Wed, 24 Jun 2026 10:50:40 +0800 Subject: [PATCH] This is an automated cherry-pick of #69331 Signed-off-by: ti-chi-bot --- DEPS.bzl | 8 + go.mod | 4 + go.sum | 5 + pkg/util/execdetails/execdetails_test.go | 72 +++++++++ pkg/util/execdetails/runtime_stats.go | 19 ++- pkg/util/execdetails/tiflash_stats.go | 178 ++++++++++++++++++++++- 6 files changed, 279 insertions(+), 7 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index 88c7e1d3c96e5..43a30aa0ce3e8 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -6582,6 +6582,7 @@ def go_deps(): name = "com_github_pingcap_tipb", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/tipb", +<<<<<<< HEAD sha256 = "2d6aaef873e175599c39f6fe3cf85cabce88a69c157ff7ec4c22fbcbc97648dd", strip_prefix = "github.com/pingcap/tipb@v0.0.0-20260210113932-1447c9d7e9fe", urls = [ @@ -6589,6 +6590,13 @@ def go_deps(): "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip", "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip", "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip", +======= + sha256 = "5a8c6a6e4487592fea5550c2a550826bd0ad7025c7bbec12365ecc7aed81db0d", + strip_prefix = "github.com/pingcap/tipb@v0.0.0-20260617071407-7c071244534b", + urls = [ + "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260617071407-7c071244534b.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260617071407-7c071244534b.zip", +>>>>>>> 9f419ca60ca (util: add support columnar scan details (#69331)) ], ) go_repository( diff --git a/go.mod b/go.mod index 2111f24b49936..ec3b29a403a70 100644 --- a/go.mod +++ b/go.mod @@ -106,7 +106,11 @@ require ( github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e +<<<<<<< HEAD github.com/pingcap/tipb v0.0.0-20260210113932-1447c9d7e9fe +======= + github.com/pingcap/tipb v0.0.0-20260617071407-7c071244534b +>>>>>>> 9f419ca60ca (util: add support columnar scan details (#69331)) github.com/prometheus/client_golang v1.23.0 github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.65.0 diff --git a/go.sum b/go.sum index 7d70bf73bdbca..ae720a631718d 100644 --- a/go.sum +++ b/go.sum @@ -738,8 +738,13 @@ github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d h1:5JCgncG9X7 github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d/go.mod h1:HMNxmg0/lrn3SPGJ6LTZqP0WwEpcXMu9s/4TWJbzT8w= github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE= github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530= +<<<<<<< HEAD github.com/pingcap/tipb v0.0.0-20260210113932-1447c9d7e9fe h1:Zmz9mON+2NoKDVjkJbk6NZbFoTzVzk8MPTbRnu+MiVM= github.com/pingcap/tipb v0.0.0-20260210113932-1447c9d7e9fe/go.mod h1:RM8iRcMalzOthG2XJxnNBniM4xFGb/lDwHUwqkaVzt4= +======= +github.com/pingcap/tipb v0.0.0-20260617071407-7c071244534b h1:10OHx2/2noFfPU1K3c3QXZshsw3XvKssk/mjbLdw8DI= +github.com/pingcap/tipb v0.0.0-20260617071407-7c071244534b/go.mod h1:RM8iRcMalzOthG2XJxnNBniM4xFGb/lDwHUwqkaVzt4= +>>>>>>> 9f419ca60ca (util: add support columnar scan details (#69331)) github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/pkg/util/execdetails/execdetails_test.go b/pkg/util/execdetails/execdetails_test.go index 224f4400e43f3..2369dda4cb37e 100644 --- a/pkg/util/execdetails/execdetails_test.go +++ b/pkg/util/execdetails/execdetails_test.go @@ -321,6 +321,38 @@ func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, Nu NumIterations: &NumIterations, Concurrency: &Concurrency, ExecutorId: &ExecutorID, DetailInfo: &tipb.ExecutorExecutionSummary_TiflashScanContext{TiflashScanContext: &tiflashScanContext}, TiflashWaitSummary: &tiflashWaitSummary, TiflashNetworkSummary: &tiflashNetworkSummary} } +func mockExecutorExecutionSummaryForTiFlashColumnar(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, regions, readTasks, physicalTables, columns, userReadBytes, mvccInputRows, mvccInputBytes, mvccOutputRows, totalReadBlockMs, totalSerializeBlockMs, totalInitReaderMs, totalPrefetchMs, roughCheckTotalPacks, roughCheckSelectedPacks, roughCheckSkippedPacks, roughCheckUnknownPacks, remoteSegments, totalSegments, totalDeserializeBlockMs uint64, ExecutorID string) *tipb.ExecutorExecutionSummary { + columnarScanContext := tipb.ColumnarScanContext{ + Regions: ®ions, + ReadTasks: &readTasks, + PhysicalTables: &physicalTables, + Columns: &columns, + UserReadBytes: &userReadBytes, + MvccInputRows: &mvccInputRows, + MvccInputBytes: &mvccInputBytes, + MvccOutputRows: &mvccOutputRows, + TotalReadBlockMs: &totalReadBlockMs, + TotalSerializeBlockMs: &totalSerializeBlockMs, + TotalInitReaderMs: &totalInitReaderMs, + TotalPrefetchMs: &totalPrefetchMs, + RoughCheckTotalPacks: &roughCheckTotalPacks, + RoughCheckSelectedPacks: &roughCheckSelectedPacks, + RoughCheckSkippedPacks: &roughCheckSkippedPacks, + RoughCheckUnknownPacks: &roughCheckUnknownPacks, + RemoteSegments: &remoteSegments, + TotalSegments: &totalSegments, + TotalDeserializeBlockMs: &totalDeserializeBlockMs, + } + return &tipb.ExecutorExecutionSummary{ + TimeProcessedNs: &TimeProcessedNs, + NumProducedRows: &NumProducedRows, + NumIterations: &NumIterations, + Concurrency: &Concurrency, + ExecutorId: &ExecutorID, + DetailInfo: &tipb.ExecutorExecutionSummary_ColumnarScanContext{ColumnarScanContext: &columnarScanContext}, + } +} + func TestCopRuntimeStats(t *testing.T) { stats := NewRuntimeStatsColl(nil) tableScanID := 1 @@ -719,6 +751,46 @@ func TestVectorSearchStats(t *testing.T) { require.Equal(t, "tiflash_task:{time:0s, loops:0, threads:0}, vector_idx:{load:{total:0ms,from_s3:1,from_disk:0,from_cache:0},search:{total:0ms,visited_nodes:0,discarded_nodes:0},read:{vec_total:0ms,others_total:0ms}}, tiflash_scan:{mvcc_input_rows:0, mvcc_input_bytes:0, mvcc_output_rows:0, local_regions:0, remote_regions:0, tot_learner_read:0ms, region_balance:none, delta_rows:0, delta_bytes:0, segments:0, stale_read_regions:0, tot_build_snapshot:0ms, tot_build_bitmap:0ms, tot_build_inputstream:0ms, min_local_stream:0ms, max_local_stream:0ms, dtfile:{data_scanned_rows:0, data_skipped_rows:0, mvcc_scanned_rows:0, mvcc_skipped_rows:0, lm_filter_scanned_rows:0, lm_filter_skipped_rows:0, tot_rs_index_check:0ms, tot_read:0ms}}", s.String()) } +func TestColumnarScanContextStats(t *testing.T) { + stats := NewRuntimeStatsColl(nil) + execSummary := mockExecutorExecutionSummaryForTiFlashColumnar( + 1, 10, 2, 1, + 2, 4, 3, 5, 2048, + 100, 4096, 80, + 7, 8, 9, 10, + 11, 12, 13, 14, + 15, 16, 17, + "tablescan_1", + ) + stats.RecordOneCopTask(1, kv.TiFlash, execSummary) + stats.RecordOneCopTask(1, kv.TiFlash, mockExecutorExecutionSummaryForTiFlashColumnar( + 2, 20, 3, 2, + 4, 6, 2, 4, 1024, + 10, 2048, 8, + 1, 2, 3, 4, + 5, 6, 7, 8, + 9, 10, 11, + "tablescan_1", + )) + s := stats.GetCopStats(1) + require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:5, tasks:2, threads:3}, columnar_scan:{mvcc_input_rows:110, mvcc_input_bytes:6144, mvcc_output_rows:88, regions:6, read_tasks:10, physical_tables:3, columns:5, user_read_bytes:3072, read_block:8ms, serialize_block:10ms, init_reader:12ms, prefetch:14ms, deserialize_block:28ms, rough_check:{total:16, selected:18, skipped:20, unknown:22}, remote_segments:24, total_segments:26}", s.String()) + + zeroStats := NewRuntimeStatsColl(nil) + zeroExecSummary := mockExecutorExecutionSummaryForTiFlashColumnar( + 1, 0, 1, 1, + 0, 0, 0, 0, 0, + 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, + "tablescan_1", + ) + zeroStats.RecordOneCopTask(1, kv.TiFlash, zeroExecSummary) + zeroString := zeroStats.GetCopStats(1).String() + require.Contains(t, zeroString, "columnar_scan:{") + require.NotContains(t, zeroString, "tiflash_scan:{") +} + func TestRuntimeStatsWithCommit(t *testing.T) { commitDetail := &util.CommitDetails{ GetCommitTsTime: time.Second, diff --git a/pkg/util/execdetails/runtime_stats.go b/pkg/util/execdetails/runtime_stats.go index ee5b2b1be377b..7dbf875c4faab 100644 --- a/pkg/util/execdetails/runtime_stats.go +++ b/pkg/util/execdetails/runtime_stats.go @@ -123,9 +123,10 @@ func (e *basicCopRuntimeStats) Clone() RuntimeStats { } if e.tiflashStats != nil { stats.tiflashStats = &TiflashStats{ - scanContext: e.tiflashStats.scanContext.Clone(), - waitSummary: e.tiflashStats.waitSummary.Clone(), - networkSummary: e.tiflashStats.networkSummary.Clone(), + scanContext: e.tiflashStats.scanContext.Clone(), + columnarScanContext: e.tiflashStats.columnarScanContext.Clone(), + waitSummary: e.tiflashStats.waitSummary.Clone(), + networkSummary: e.tiflashStats.networkSummary.Clone(), } } return stats @@ -148,6 +149,7 @@ func (e *basicCopRuntimeStats) Merge(rs RuntimeStats) { e.tiflashStats = &TiflashStats{} } e.tiflashStats.scanContext.Merge(tmp.tiflashStats.scanContext) + e.tiflashStats.columnarScanContext.Merge(tmp.tiflashStats.columnarScanContext) e.tiflashStats.waitSummary.Merge(tmp.tiflashStats.waitSummary) e.tiflashStats.networkSummary.Merge(tmp.tiflashStats.networkSummary) } @@ -165,6 +167,12 @@ func (e *basicCopRuntimeStats) mergeExecSummary(summary *tipb.ExecutorExecutionS } e.tiflashStats.scanContext.mergeExecSummary(tiflashScanContext) } + if columnarScanContext := summary.GetColumnarScanContext(); columnarScanContext != nil { + if e.tiflashStats == nil { + e.tiflashStats = &TiflashStats{} + } + e.tiflashStats.columnarScanContext.mergeExecSummary(columnarScanContext) + } if tiflashWaitSummary := summary.GetTiflashWaitSummary(); tiflashWaitSummary != nil { if e.tiflashStats == nil { e.tiflashStats = &TiflashStats{} @@ -246,7 +254,10 @@ func (crs *CopRuntimeStats) String() string { buf.WriteString(", ") buf.WriteString(crs.stats.tiflashStats.networkSummary.String()) } - if !crs.stats.tiflashStats.scanContext.Empty() { + if !crs.stats.tiflashStats.columnarScanContext.Empty() { + buf.WriteString(", ") + buf.WriteString(crs.stats.tiflashStats.columnarScanContext.String()) + } else if !crs.stats.tiflashStats.scanContext.Empty() { buf.WriteString(", ") buf.WriteString(crs.stats.tiflashStats.scanContext.String()) } diff --git a/pkg/util/execdetails/tiflash_stats.go b/pkg/util/execdetails/tiflash_stats.go index 58c4ec8dd0d05..106e03629fbaf 100644 --- a/pkg/util/execdetails/tiflash_stats.go +++ b/pkg/util/execdetails/tiflash_stats.go @@ -31,9 +31,34 @@ import ( // TiflashStats contains tiflash execution stats. type TiflashStats struct { - scanContext TiFlashScanContext - waitSummary TiFlashWaitSummary - networkSummary TiFlashNetworkTrafficSummary + scanContext TiFlashScanContext + columnarScanContext TiFlashColumnarScanContext + waitSummary TiFlashWaitSummary + networkSummary TiFlashNetworkTrafficSummary +} + +// TiFlashColumnarScanContext is used to express the table scan information in tiflash columnar read path. +type TiFlashColumnarScanContext struct { + hasStats bool + regions uint64 + readTasks uint64 + physicalTables uint64 + columns uint64 + userReadBytes uint64 + mvccInputRows uint64 + mvccInputBytes uint64 + mvccOutputRows uint64 + totalReadBlockMs uint64 + totalSerializeBlockMs uint64 + totalInitReaderMs uint64 + totalPrefetchMs uint64 + roughCheckTotalPacks uint64 + roughCheckSelectedPacks uint64 + roughCheckSkippedPacks uint64 + roughCheckUnknownPacks uint64 + remoteSegments uint64 + totalSegments uint64 + totalDeserializeBlockMs uint64 } // TiFlashScanContext is used to express the table scan information in tiflash @@ -534,6 +559,153 @@ func (context *TiFlashScanContext) Empty() bool { return res } +// Clone implements the deep copy of * TiFlashColumnarScanContext +func (context *TiFlashColumnarScanContext) Clone() TiFlashColumnarScanContext { + return TiFlashColumnarScanContext{ + hasStats: context.hasStats, + regions: context.regions, + readTasks: context.readTasks, + physicalTables: context.physicalTables, + columns: context.columns, + userReadBytes: context.userReadBytes, + mvccInputRows: context.mvccInputRows, + mvccInputBytes: context.mvccInputBytes, + mvccOutputRows: context.mvccOutputRows, + totalReadBlockMs: context.totalReadBlockMs, + totalSerializeBlockMs: context.totalSerializeBlockMs, + totalInitReaderMs: context.totalInitReaderMs, + totalPrefetchMs: context.totalPrefetchMs, + roughCheckTotalPacks: context.roughCheckTotalPacks, + roughCheckSelectedPacks: context.roughCheckSelectedPacks, + roughCheckSkippedPacks: context.roughCheckSkippedPacks, + roughCheckUnknownPacks: context.roughCheckUnknownPacks, + remoteSegments: context.remoteSegments, + totalSegments: context.totalSegments, + totalDeserializeBlockMs: context.totalDeserializeBlockMs, + } +} + +func (context *TiFlashColumnarScanContext) String() string { + return fmt.Sprintf("columnar_scan:{"+ + "mvcc_input_rows:%d, "+ + "mvcc_input_bytes:%d, "+ + "mvcc_output_rows:%d, "+ + "regions:%d, "+ + "read_tasks:%d, "+ + "physical_tables:%d, "+ + "columns:%d, "+ + "user_read_bytes:%d, "+ + "read_block:%dms, "+ + "serialize_block:%dms, "+ + "init_reader:%dms, "+ + "prefetch:%dms, "+ + "deserialize_block:%dms, "+ + "rough_check:{total:%d, selected:%d, skipped:%d, unknown:%d}, "+ + "remote_segments:%d, "+ + "total_segments:%d}", + context.mvccInputRows, + context.mvccInputBytes, + context.mvccOutputRows, + context.regions, + context.readTasks, + context.physicalTables, + context.columns, + context.userReadBytes, + context.totalReadBlockMs, + context.totalSerializeBlockMs, + context.totalInitReaderMs, + context.totalPrefetchMs, + context.totalDeserializeBlockMs, + context.roughCheckTotalPacks, + context.roughCheckSelectedPacks, + context.roughCheckSkippedPacks, + context.roughCheckUnknownPacks, + context.remoteSegments, + context.totalSegments) +} + +// Merge make sum to merge the information in TiFlashColumnarScanContext +func (context *TiFlashColumnarScanContext) Merge(other TiFlashColumnarScanContext) { + context.hasStats = context.hasStats || other.hasStats + context.regions += other.regions + context.readTasks += other.readTasks + if other.physicalTables > context.physicalTables { + context.physicalTables = other.physicalTables + } + if other.columns > context.columns { + context.columns = other.columns + } + context.userReadBytes += other.userReadBytes + context.mvccInputRows += other.mvccInputRows + context.mvccInputBytes += other.mvccInputBytes + context.mvccOutputRows += other.mvccOutputRows + context.totalReadBlockMs += other.totalReadBlockMs + context.totalSerializeBlockMs += other.totalSerializeBlockMs + context.totalInitReaderMs += other.totalInitReaderMs + context.totalPrefetchMs += other.totalPrefetchMs + context.roughCheckTotalPacks += other.roughCheckTotalPacks + context.roughCheckSelectedPacks += other.roughCheckSelectedPacks + context.roughCheckSkippedPacks += other.roughCheckSkippedPacks + context.roughCheckUnknownPacks += other.roughCheckUnknownPacks + context.remoteSegments += other.remoteSegments + context.totalSegments += other.totalSegments + context.totalDeserializeBlockMs += other.totalDeserializeBlockMs +} + +func (context *TiFlashColumnarScanContext) mergeExecSummary(summary *tipb.ColumnarScanContext) { + if summary == nil { + return + } + context.hasStats = true + context.regions += summary.GetRegions() + context.readTasks += summary.GetReadTasks() + if summary.GetPhysicalTables() > context.physicalTables { + context.physicalTables = summary.GetPhysicalTables() + } + if summary.GetColumns() > context.columns { + context.columns = summary.GetColumns() + } + context.userReadBytes += summary.GetUserReadBytes() + context.mvccInputRows += summary.GetMvccInputRows() + context.mvccInputBytes += summary.GetMvccInputBytes() + context.mvccOutputRows += summary.GetMvccOutputRows() + context.totalReadBlockMs += summary.GetTotalReadBlockMs() + context.totalSerializeBlockMs += summary.GetTotalSerializeBlockMs() + context.totalInitReaderMs += summary.GetTotalInitReaderMs() + context.totalPrefetchMs += summary.GetTotalPrefetchMs() + context.roughCheckTotalPacks += summary.GetRoughCheckTotalPacks() + context.roughCheckSelectedPacks += summary.GetRoughCheckSelectedPacks() + context.roughCheckSkippedPacks += summary.GetRoughCheckSkippedPacks() + context.roughCheckUnknownPacks += summary.GetRoughCheckUnknownPacks() + context.remoteSegments += summary.GetRemoteSegments() + context.totalSegments += summary.GetTotalSegments() + context.totalDeserializeBlockMs += summary.GetTotalDeserializeBlockMs() +} + +// Empty check whether TiFlashColumnarScanContext is empty. +func (context *TiFlashColumnarScanContext) Empty() bool { + return !context.hasStats && + context.regions == 0 && + context.readTasks == 0 && + context.physicalTables == 0 && + context.columns == 0 && + context.userReadBytes == 0 && + context.mvccInputRows == 0 && + context.mvccInputBytes == 0 && + context.mvccOutputRows == 0 && + context.totalReadBlockMs == 0 && + context.totalSerializeBlockMs == 0 && + context.totalInitReaderMs == 0 && + context.totalPrefetchMs == 0 && + context.roughCheckTotalPacks == 0 && + context.roughCheckSelectedPacks == 0 && + context.roughCheckSkippedPacks == 0 && + context.roughCheckUnknownPacks == 0 && + context.remoteSegments == 0 && + context.totalSegments == 0 && + context.totalDeserializeBlockMs == 0 +} + // TiFlashWaitSummary is used to express all kinds of wait information in tiflash type TiFlashWaitSummary struct { // keep execution time to do merge work, always record the wait time with largest execution time