Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions DEPS.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -6582,13 +6582,21 @@ def go_deps():
name = "com_github_pingcap_tipb",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/tipb",
<<<<<<< HEAD
sha256 = "2d6aaef873e175599c39f6fe3cf85cabce88a69c157ff7ec4c22fbcbc97648dd",
strip_prefix = "github.com/pingcap/tipb@v0.0.0-20260210113932-1447c9d7e9fe",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
"http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
"https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
=======
sha256 = "5a8c6a6e4487592fea5550c2a550826bd0ad7025c7bbec12365ecc7aed81db0d",
strip_prefix = "github.com/pingcap/tipb@v0.0.0-20260617071407-7c071244534b",
urls = [
"https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260617071407-7c071244534b.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260617071407-7c071244534b.zip",
>>>>>>> 9f419ca60ca (util: add support columnar scan details (#69331))
Comment on lines +6585 to +6599

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🔴 Critical | ⚡ Quick win

Fix the unresolved conflict in the Bazel tipb pin.

This go_repository stanza is currently invalid Starlark because of the conflict markers, so Bazel cannot resolve @com_github_pingcap_tipb//go-tipb. Resolve it to one sha256/strip_prefix/urls set that matches the final go.mod version.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@DEPS.bzl` around lines 6585 - 6599, Resolve the unresolved merge conflict in
the tipb go_repository stanza by removing the conflict markers and keeping a
single consistent set of sha256, strip_prefix, and urls in DEPS.bzl; ensure the
selected version matches the intended final go.mod pin so
`@com_github_pingcap_tipb//go-tipb` remains valid.

],
)
go_repository(
Expand Down
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,11 @@ require (
github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5
github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e
<<<<<<< HEAD
github.com/pingcap/tipb v0.0.0-20260210113932-1447c9d7e9fe
=======
github.com/pingcap/tipb v0.0.0-20260617071407-7c071244534b
>>>>>>> 9f419ca60ca (util: add support columnar scan details (#69331))
Comment on lines +109 to +113

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🔴 Critical | ⚡ Quick win

Resolve the tipb merge conflict before merging.

These conflict markers make go.mod unparsable, and this PR also needs the resolved github.com/pingcap/tipb revision to be the one that exposes ColumnarScanContext used by pkg/util/execdetails/runtime_stats.go and pkg/util/execdetails/execdetails_test.go. Please collapse this to a single version and keep it consistent with DEPS.bzl.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@go.mod` around lines 109 - 113, Resolve the merge conflict in go.mod by
removing the conflict markers and leaving a single github.com/pingcap/tipb
version; choose the revision that includes ColumnarScanContext so
pkg/util/execdetails/runtime_stats.go and
pkg/util/execdetails/execdetails_test.go stay compatible. After updating go.mod,
make sure the same tipb revision is reflected consistently in DEPS.bzl.

github.com/prometheus/client_golang v1.23.0
github.com/prometheus/client_model v0.6.2
github.com/prometheus/common v0.65.0
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -738,8 +738,13 @@ github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d h1:5JCgncG9X7
github.com/pingcap/metering_sdk v0.0.0-20260324055927-14fead745f1d/go.mod h1:HMNxmg0/lrn3SPGJ6LTZqP0WwEpcXMu9s/4TWJbzT8w=
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE=
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530=
<<<<<<< HEAD
github.com/pingcap/tipb v0.0.0-20260210113932-1447c9d7e9fe h1:Zmz9mON+2NoKDVjkJbk6NZbFoTzVzk8MPTbRnu+MiVM=
github.com/pingcap/tipb v0.0.0-20260210113932-1447c9d7e9fe/go.mod h1:RM8iRcMalzOthG2XJxnNBniM4xFGb/lDwHUwqkaVzt4=
=======
github.com/pingcap/tipb v0.0.0-20260617071407-7c071244534b h1:10OHx2/2noFfPU1K3c3QXZshsw3XvKssk/mjbLdw8DI=
github.com/pingcap/tipb v0.0.0-20260617071407-7c071244534b/go.mod h1:RM8iRcMalzOthG2XJxnNBniM4xFGb/lDwHUwqkaVzt4=
>>>>>>> 9f419ca60ca (util: add support columnar scan details (#69331))
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
Expand Down
72 changes: 72 additions & 0 deletions pkg/util/execdetails/execdetails_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,38 @@ func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, Nu
NumIterations: &NumIterations, Concurrency: &Concurrency, ExecutorId: &ExecutorID, DetailInfo: &tipb.ExecutorExecutionSummary_TiflashScanContext{TiflashScanContext: &tiflashScanContext}, TiflashWaitSummary: &tiflashWaitSummary, TiflashNetworkSummary: &tiflashNetworkSummary}
}

func mockExecutorExecutionSummaryForTiFlashColumnar(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, regions, readTasks, physicalTables, columns, userReadBytes, mvccInputRows, mvccInputBytes, mvccOutputRows, totalReadBlockMs, totalSerializeBlockMs, totalInitReaderMs, totalPrefetchMs, roughCheckTotalPacks, roughCheckSelectedPacks, roughCheckSkippedPacks, roughCheckUnknownPacks, remoteSegments, totalSegments, totalDeserializeBlockMs uint64, ExecutorID string) *tipb.ExecutorExecutionSummary {
columnarScanContext := tipb.ColumnarScanContext{
Regions: &regions,
ReadTasks: &readTasks,
PhysicalTables: &physicalTables,
Columns: &columns,
UserReadBytes: &userReadBytes,
MvccInputRows: &mvccInputRows,
MvccInputBytes: &mvccInputBytes,
MvccOutputRows: &mvccOutputRows,
TotalReadBlockMs: &totalReadBlockMs,
TotalSerializeBlockMs: &totalSerializeBlockMs,
TotalInitReaderMs: &totalInitReaderMs,
TotalPrefetchMs: &totalPrefetchMs,
RoughCheckTotalPacks: &roughCheckTotalPacks,
RoughCheckSelectedPacks: &roughCheckSelectedPacks,
RoughCheckSkippedPacks: &roughCheckSkippedPacks,
RoughCheckUnknownPacks: &roughCheckUnknownPacks,
RemoteSegments: &remoteSegments,
TotalSegments: &totalSegments,
TotalDeserializeBlockMs: &totalDeserializeBlockMs,
}
return &tipb.ExecutorExecutionSummary{
TimeProcessedNs: &TimeProcessedNs,
NumProducedRows: &NumProducedRows,
NumIterations: &NumIterations,
Concurrency: &Concurrency,
ExecutorId: &ExecutorID,
DetailInfo: &tipb.ExecutorExecutionSummary_ColumnarScanContext{ColumnarScanContext: &columnarScanContext},
}
}

func TestCopRuntimeStats(t *testing.T) {
stats := NewRuntimeStatsColl(nil)
tableScanID := 1
Expand Down Expand Up @@ -719,6 +751,46 @@ func TestVectorSearchStats(t *testing.T) {
require.Equal(t, "tiflash_task:{time:0s, loops:0, threads:0}, vector_idx:{load:{total:0ms,from_s3:1,from_disk:0,from_cache:0},search:{total:0ms,visited_nodes:0,discarded_nodes:0},read:{vec_total:0ms,others_total:0ms}}, tiflash_scan:{mvcc_input_rows:0, mvcc_input_bytes:0, mvcc_output_rows:0, local_regions:0, remote_regions:0, tot_learner_read:0ms, region_balance:none, delta_rows:0, delta_bytes:0, segments:0, stale_read_regions:0, tot_build_snapshot:0ms, tot_build_bitmap:0ms, tot_build_inputstream:0ms, min_local_stream:0ms, max_local_stream:0ms, dtfile:{data_scanned_rows:0, data_skipped_rows:0, mvcc_scanned_rows:0, mvcc_skipped_rows:0, lm_filter_scanned_rows:0, lm_filter_skipped_rows:0, tot_rs_index_check:0ms, tot_read:0ms}}", s.String())
}

func TestColumnarScanContextStats(t *testing.T) {
stats := NewRuntimeStatsColl(nil)
execSummary := mockExecutorExecutionSummaryForTiFlashColumnar(
1, 10, 2, 1,
2, 4, 3, 5, 2048,
100, 4096, 80,
7, 8, 9, 10,
11, 12, 13, 14,
15, 16, 17,
"tablescan_1",
)
stats.RecordOneCopTask(1, kv.TiFlash, execSummary)
stats.RecordOneCopTask(1, kv.TiFlash, mockExecutorExecutionSummaryForTiFlashColumnar(
2, 20, 3, 2,
4, 6, 2, 4, 1024,
10, 2048, 8,
1, 2, 3, 4,
5, 6, 7, 8,
9, 10, 11,
"tablescan_1",
))
s := stats.GetCopStats(1)
require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:5, tasks:2, threads:3}, columnar_scan:{mvcc_input_rows:110, mvcc_input_bytes:6144, mvcc_output_rows:88, regions:6, read_tasks:10, physical_tables:3, columns:5, user_read_bytes:3072, read_block:8ms, serialize_block:10ms, init_reader:12ms, prefetch:14ms, deserialize_block:28ms, rough_check:{total:16, selected:18, skipped:20, unknown:22}, remote_segments:24, total_segments:26}", s.String())

zeroStats := NewRuntimeStatsColl(nil)
zeroExecSummary := mockExecutorExecutionSummaryForTiFlashColumnar(
1, 0, 1, 1,
0, 0, 0, 0, 0,
0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0,
"tablescan_1",
)
zeroStats.RecordOneCopTask(1, kv.TiFlash, zeroExecSummary)
zeroString := zeroStats.GetCopStats(1).String()
require.Contains(t, zeroString, "columnar_scan:{")
require.NotContains(t, zeroString, "tiflash_scan:{")
}

func TestRuntimeStatsWithCommit(t *testing.T) {
commitDetail := &util.CommitDetails{
GetCommitTsTime: time.Second,
Expand Down
19 changes: 15 additions & 4 deletions pkg/util/execdetails/runtime_stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,10 @@ func (e *basicCopRuntimeStats) Clone() RuntimeStats {
}
if e.tiflashStats != nil {
stats.tiflashStats = &TiflashStats{
scanContext: e.tiflashStats.scanContext.Clone(),
waitSummary: e.tiflashStats.waitSummary.Clone(),
networkSummary: e.tiflashStats.networkSummary.Clone(),
scanContext: e.tiflashStats.scanContext.Clone(),
columnarScanContext: e.tiflashStats.columnarScanContext.Clone(),
waitSummary: e.tiflashStats.waitSummary.Clone(),
networkSummary: e.tiflashStats.networkSummary.Clone(),
}
}
return stats
Expand All @@ -148,6 +149,7 @@ func (e *basicCopRuntimeStats) Merge(rs RuntimeStats) {
e.tiflashStats = &TiflashStats{}
}
e.tiflashStats.scanContext.Merge(tmp.tiflashStats.scanContext)
e.tiflashStats.columnarScanContext.Merge(tmp.tiflashStats.columnarScanContext)
e.tiflashStats.waitSummary.Merge(tmp.tiflashStats.waitSummary)
e.tiflashStats.networkSummary.Merge(tmp.tiflashStats.networkSummary)
}
Expand All @@ -165,6 +167,12 @@ func (e *basicCopRuntimeStats) mergeExecSummary(summary *tipb.ExecutorExecutionS
}
e.tiflashStats.scanContext.mergeExecSummary(tiflashScanContext)
}
if columnarScanContext := summary.GetColumnarScanContext(); columnarScanContext != nil {
if e.tiflashStats == nil {
e.tiflashStats = &TiflashStats{}
}
e.tiflashStats.columnarScanContext.mergeExecSummary(columnarScanContext)
}
if tiflashWaitSummary := summary.GetTiflashWaitSummary(); tiflashWaitSummary != nil {
if e.tiflashStats == nil {
e.tiflashStats = &TiflashStats{}
Expand Down Expand Up @@ -246,7 +254,10 @@ func (crs *CopRuntimeStats) String() string {
buf.WriteString(", ")
buf.WriteString(crs.stats.tiflashStats.networkSummary.String())
}
if !crs.stats.tiflashStats.scanContext.Empty() {
if !crs.stats.tiflashStats.columnarScanContext.Empty() {
buf.WriteString(", ")
buf.WriteString(crs.stats.tiflashStats.columnarScanContext.String())
} else if !crs.stats.tiflashStats.scanContext.Empty() {
buf.WriteString(", ")
buf.WriteString(crs.stats.tiflashStats.scanContext.String())
}
Expand Down
178 changes: 175 additions & 3 deletions pkg/util/execdetails/tiflash_stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,34 @@ import (

// TiflashStats contains tiflash execution stats.
type TiflashStats struct {
scanContext TiFlashScanContext
waitSummary TiFlashWaitSummary
networkSummary TiFlashNetworkTrafficSummary
scanContext TiFlashScanContext
columnarScanContext TiFlashColumnarScanContext
waitSummary TiFlashWaitSummary
networkSummary TiFlashNetworkTrafficSummary
}

// TiFlashColumnarScanContext is used to express the table scan information in tiflash columnar read path.
type TiFlashColumnarScanContext struct {
hasStats bool
regions uint64
readTasks uint64
physicalTables uint64
columns uint64
userReadBytes uint64
mvccInputRows uint64
mvccInputBytes uint64
mvccOutputRows uint64
totalReadBlockMs uint64
totalSerializeBlockMs uint64
totalInitReaderMs uint64
totalPrefetchMs uint64
roughCheckTotalPacks uint64
roughCheckSelectedPacks uint64
roughCheckSkippedPacks uint64
roughCheckUnknownPacks uint64
remoteSegments uint64
totalSegments uint64
totalDeserializeBlockMs uint64
}

// TiFlashScanContext is used to express the table scan information in tiflash
Expand Down Expand Up @@ -534,6 +559,153 @@ func (context *TiFlashScanContext) Empty() bool {
return res
}

// Clone implements the deep copy of * TiFlashColumnarScanContext
func (context *TiFlashColumnarScanContext) Clone() TiFlashColumnarScanContext {
return TiFlashColumnarScanContext{
hasStats: context.hasStats,
regions: context.regions,
readTasks: context.readTasks,
physicalTables: context.physicalTables,
columns: context.columns,
userReadBytes: context.userReadBytes,
mvccInputRows: context.mvccInputRows,
mvccInputBytes: context.mvccInputBytes,
mvccOutputRows: context.mvccOutputRows,
totalReadBlockMs: context.totalReadBlockMs,
totalSerializeBlockMs: context.totalSerializeBlockMs,
totalInitReaderMs: context.totalInitReaderMs,
totalPrefetchMs: context.totalPrefetchMs,
roughCheckTotalPacks: context.roughCheckTotalPacks,
roughCheckSelectedPacks: context.roughCheckSelectedPacks,
roughCheckSkippedPacks: context.roughCheckSkippedPacks,
roughCheckUnknownPacks: context.roughCheckUnknownPacks,
remoteSegments: context.remoteSegments,
totalSegments: context.totalSegments,
totalDeserializeBlockMs: context.totalDeserializeBlockMs,
}
}

func (context *TiFlashColumnarScanContext) String() string {
return fmt.Sprintf("columnar_scan:{"+
"mvcc_input_rows:%d, "+
"mvcc_input_bytes:%d, "+
"mvcc_output_rows:%d, "+
"regions:%d, "+
"read_tasks:%d, "+
"physical_tables:%d, "+
"columns:%d, "+
"user_read_bytes:%d, "+
"read_block:%dms, "+
"serialize_block:%dms, "+
"init_reader:%dms, "+
"prefetch:%dms, "+
"deserialize_block:%dms, "+
"rough_check:{total:%d, selected:%d, skipped:%d, unknown:%d}, "+
"remote_segments:%d, "+
"total_segments:%d}",
context.mvccInputRows,
context.mvccInputBytes,
context.mvccOutputRows,
context.regions,
context.readTasks,
context.physicalTables,
context.columns,
context.userReadBytes,
context.totalReadBlockMs,
context.totalSerializeBlockMs,
context.totalInitReaderMs,
context.totalPrefetchMs,
context.totalDeserializeBlockMs,
context.roughCheckTotalPacks,
context.roughCheckSelectedPacks,
context.roughCheckSkippedPacks,
context.roughCheckUnknownPacks,
context.remoteSegments,
context.totalSegments)
}

// Merge make sum to merge the information in TiFlashColumnarScanContext
func (context *TiFlashColumnarScanContext) Merge(other TiFlashColumnarScanContext) {
context.hasStats = context.hasStats || other.hasStats
context.regions += other.regions
context.readTasks += other.readTasks
if other.physicalTables > context.physicalTables {
context.physicalTables = other.physicalTables
}
if other.columns > context.columns {
context.columns = other.columns
}
context.userReadBytes += other.userReadBytes
context.mvccInputRows += other.mvccInputRows
context.mvccInputBytes += other.mvccInputBytes
context.mvccOutputRows += other.mvccOutputRows
context.totalReadBlockMs += other.totalReadBlockMs
context.totalSerializeBlockMs += other.totalSerializeBlockMs
context.totalInitReaderMs += other.totalInitReaderMs
context.totalPrefetchMs += other.totalPrefetchMs
context.roughCheckTotalPacks += other.roughCheckTotalPacks
context.roughCheckSelectedPacks += other.roughCheckSelectedPacks
context.roughCheckSkippedPacks += other.roughCheckSkippedPacks
context.roughCheckUnknownPacks += other.roughCheckUnknownPacks
context.remoteSegments += other.remoteSegments
context.totalSegments += other.totalSegments
context.totalDeserializeBlockMs += other.totalDeserializeBlockMs
}

func (context *TiFlashColumnarScanContext) mergeExecSummary(summary *tipb.ColumnarScanContext) {
if summary == nil {
return
}
context.hasStats = true
context.regions += summary.GetRegions()
context.readTasks += summary.GetReadTasks()
if summary.GetPhysicalTables() > context.physicalTables {
context.physicalTables = summary.GetPhysicalTables()
}
if summary.GetColumns() > context.columns {
context.columns = summary.GetColumns()
}
context.userReadBytes += summary.GetUserReadBytes()
context.mvccInputRows += summary.GetMvccInputRows()
context.mvccInputBytes += summary.GetMvccInputBytes()
context.mvccOutputRows += summary.GetMvccOutputRows()
context.totalReadBlockMs += summary.GetTotalReadBlockMs()
context.totalSerializeBlockMs += summary.GetTotalSerializeBlockMs()
context.totalInitReaderMs += summary.GetTotalInitReaderMs()
context.totalPrefetchMs += summary.GetTotalPrefetchMs()
context.roughCheckTotalPacks += summary.GetRoughCheckTotalPacks()
context.roughCheckSelectedPacks += summary.GetRoughCheckSelectedPacks()
context.roughCheckSkippedPacks += summary.GetRoughCheckSkippedPacks()
context.roughCheckUnknownPacks += summary.GetRoughCheckUnknownPacks()
context.remoteSegments += summary.GetRemoteSegments()
context.totalSegments += summary.GetTotalSegments()
context.totalDeserializeBlockMs += summary.GetTotalDeserializeBlockMs()
}

// Empty check whether TiFlashColumnarScanContext is empty.
func (context *TiFlashColumnarScanContext) Empty() bool {
return !context.hasStats &&
context.regions == 0 &&
context.readTasks == 0 &&
context.physicalTables == 0 &&
context.columns == 0 &&
context.userReadBytes == 0 &&
context.mvccInputRows == 0 &&
context.mvccInputBytes == 0 &&
context.mvccOutputRows == 0 &&
context.totalReadBlockMs == 0 &&
context.totalSerializeBlockMs == 0 &&
context.totalInitReaderMs == 0 &&
context.totalPrefetchMs == 0 &&
context.roughCheckTotalPacks == 0 &&
context.roughCheckSelectedPacks == 0 &&
context.roughCheckSkippedPacks == 0 &&
context.roughCheckUnknownPacks == 0 &&
context.remoteSegments == 0 &&
context.totalSegments == 0 &&
context.totalDeserializeBlockMs == 0
}

// TiFlashWaitSummary is used to express all kinds of wait information in tiflash
type TiFlashWaitSummary struct {
// keep execution time to do merge work, always record the wait time with largest execution time
Expand Down