Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Explain: Add new item in scan context #45543

Merged
merged 10 commits into from
Aug 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions DEPS.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -5780,13 +5780,13 @@ def go_deps():
name = "com_github_pingcap_tipb",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/tipb",
sha256 = "bb5d3cf24245a26acb1c8f8c110eb3313f21a655a8b49280f2772082bb29dc22",
strip_prefix = "github.com/pingcap/[email protected]20230607071926-bda24015c2d6",
sha256 = "74b710db49c16c0400c44d14e7d2e23257d094d7cf39c71e5715bb5ac59fd57a",
strip_prefix = "github.com/pingcap/[email protected]20230802082933-256c189cd860",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230607071926-bda24015c2d6.zip",
"http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230607071926-bda24015c2d6.zip",
"https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230607071926-bda24015c2d6.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230607071926-bda24015c2d6.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230802082933-256c189cd860.zip",
"http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230802082933-256c189cd860.zip",
"https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230802082933-256c189cd860.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230802082933-256c189cd860.zip",
],
)
go_repository(
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ require (
github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22
github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21
github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e
github.com/pingcap/tipb v0.0.0-20230607071926-bda24015c2d6
github.com/pingcap/tipb v0.0.0-20230802082933-256c189cd860
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.16.0
github.com/prometheus/client_model v0.4.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -816,8 +816,8 @@ github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 h1:2SOzvGvE8beiC1Y4g
github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I=
github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM=
github.com/pingcap/tipb v0.0.0-20230607071926-bda24015c2d6 h1:D79RE4RVhq2ic8sqDSv7QdL0tT5aZV3CaCXUAT41iWc=
github.com/pingcap/tipb v0.0.0-20230607071926-bda24015c2d6/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
github.com/pingcap/tipb v0.0.0-20230802082933-256c189cd860 h1:2DXvgx8MGCJitU7/IaUC+2onDAuqB7B/KMyxxiX3Y5I=
github.com/pingcap/tipb v0.0.0-20230802082933-256c189cd860/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
Expand Down
70 changes: 41 additions & 29 deletions util/execdetails/execdetails.go
Original file line number Diff line number Diff line change
Expand Up @@ -583,15 +583,18 @@ func (crs *CopRuntimeStats) RecordOneCopTask(address string, summary *tipb.Execu
storeType: crs.storeType,
BasicRuntimeStats: BasicRuntimeStats{
tiflashScanContext: TiFlashScanContext{
totalDmfileScannedPacks: summary.GetTiflashScanContext().GetTotalDmfileScannedPacks(),
totalDmfileSkippedPacks: summary.GetTiflashScanContext().GetTotalDmfileSkippedPacks(),
totalDmfileScannedRows: summary.GetTiflashScanContext().GetTotalDmfileScannedRows(),
totalDmfileSkippedRows: summary.GetTiflashScanContext().GetTotalDmfileSkippedRows(),
totalDmfileRoughSetIndexLoadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileRoughSetIndexLoadTimeMs(),
totalDmfileReadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileReadTimeMs(),
totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs(),
totalLocalRegionNum: summary.GetTiflashScanContext().GetTotalLocalRegionNum(),
totalRemoteRegionNum: summary.GetTiflashScanContext().GetTotalRemoteRegionNum()}}, threads: int32(summary.GetConcurrency()),
totalDmfileScannedPacks: summary.GetTiflashScanContext().GetTotalDmfileScannedPacks(),
totalDmfileSkippedPacks: summary.GetTiflashScanContext().GetTotalDmfileSkippedPacks(),
totalDmfileScannedRows: summary.GetTiflashScanContext().GetTotalDmfileScannedRows(),
totalDmfileSkippedRows: summary.GetTiflashScanContext().GetTotalDmfileSkippedRows(),
totalDmfileRoughSetIndexCheckTimeMs: summary.GetTiflashScanContext().GetTotalDmfileRoughSetIndexCheckTimeMs(),
totalDmfileReadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileReadTimeMs(),
totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs(),
totalLocalRegionNum: summary.GetTiflashScanContext().GetTotalLocalRegionNum(),
totalRemoteRegionNum: summary.GetTiflashScanContext().GetTotalRemoteRegionNum(),
totalLearnerReadMs: summary.GetTiflashScanContext().GetTotalLearnerReadMs(),
totalDisaggReadCacheHitSize: summary.GetTiflashScanContext().GetTotalDisaggReadCacheHitSize(),
totalDisaggReadCacheMissSize: summary.GetTiflashScanContext().GetTotalDisaggReadCacheMissSize()}}, threads: int32(summary.GetConcurrency()),
totalTasks: 1,
}
data.BasicRuntimeStats.loop.Store(int32(*summary.NumIterations))
Expand Down Expand Up @@ -716,33 +719,39 @@ type RuntimeStats interface {

// TiFlashScanContext is used to express the table scan information in tiflash
type TiFlashScanContext struct {
totalDmfileScannedPacks uint64
totalDmfileScannedRows uint64
totalDmfileSkippedPacks uint64
totalDmfileSkippedRows uint64
totalDmfileRoughSetIndexLoadTimeMs uint64
totalDmfileReadTimeMs uint64
totalCreateSnapshotTimeMs uint64
totalLocalRegionNum uint64
totalRemoteRegionNum uint64
totalDmfileScannedPacks uint64
totalDmfileScannedRows uint64
totalDmfileSkippedPacks uint64
totalDmfileSkippedRows uint64
totalDmfileRoughSetIndexCheckTimeMs uint64
totalDmfileReadTimeMs uint64
totalCreateSnapshotTimeMs uint64
totalLocalRegionNum uint64
totalRemoteRegionNum uint64
totalLearnerReadMs uint64
totalDisaggReadCacheHitSize uint64
totalDisaggReadCacheMissSize uint64
}

// Clone implements the deep copy of * TiFlashshScanContext
func (context *TiFlashScanContext) Clone() TiFlashScanContext {
return TiFlashScanContext{
totalDmfileScannedPacks: context.totalDmfileScannedPacks,
totalDmfileScannedRows: context.totalDmfileScannedRows,
totalDmfileSkippedPacks: context.totalDmfileSkippedPacks,
totalDmfileSkippedRows: context.totalDmfileSkippedRows,
totalDmfileRoughSetIndexLoadTimeMs: context.totalDmfileRoughSetIndexLoadTimeMs,
totalDmfileReadTimeMs: context.totalDmfileReadTimeMs,
totalCreateSnapshotTimeMs: context.totalCreateSnapshotTimeMs,
totalLocalRegionNum: context.totalLocalRegionNum,
totalRemoteRegionNum: context.totalRemoteRegionNum,
totalDmfileScannedPacks: context.totalDmfileScannedPacks,
totalDmfileScannedRows: context.totalDmfileScannedRows,
totalDmfileSkippedPacks: context.totalDmfileSkippedPacks,
totalDmfileSkippedRows: context.totalDmfileSkippedRows,
totalDmfileRoughSetIndexCheckTimeMs: context.totalDmfileRoughSetIndexCheckTimeMs,
totalDmfileReadTimeMs: context.totalDmfileReadTimeMs,
totalCreateSnapshotTimeMs: context.totalCreateSnapshotTimeMs,
totalLocalRegionNum: context.totalLocalRegionNum,
totalRemoteRegionNum: context.totalRemoteRegionNum,
totalLearnerReadMs: context.totalLearnerReadMs,
totalDisaggReadCacheHitSize: context.totalDisaggReadCacheHitSize,
totalDisaggReadCacheMissSize: context.totalDisaggReadCacheMissSize,
}
}
func (context *TiFlashScanContext) String() string {
return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_load_time: %dms, total_read_time: %dms}, total_create_snapshot_time: %dms, total_local_region_num: %d, total_remote_region_num: %d}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexLoadTimeMs, context.totalDmfileReadTimeMs, context.totalCreateSnapshotTimeMs, context.totalLocalRegionNum, context.totalRemoteRegionNum)
return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_check_time: %dms, total_read_time: %dms, total_disagg_read_cache_hit_size: %d, total_disagg_read_cache_miss_size: %d}, total_create_snapshot_time: %dms, total_local_region_num: %d, total_remote_region_num: %d, total_learner_read_time: %dms}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexCheckTimeMs, context.totalDmfileReadTimeMs, context.totalDisaggReadCacheHitSize, context.totalDisaggReadCacheMissSize, context.totalCreateSnapshotTimeMs, context.totalLocalRegionNum, context.totalRemoteRegionNum, context.totalLearnerReadMs)
}

// Merge make sum to merge the information in TiFlashScanContext
Expand All @@ -751,11 +760,14 @@ func (context *TiFlashScanContext) Merge(other TiFlashScanContext) {
context.totalDmfileScannedRows += other.totalDmfileScannedRows
context.totalDmfileSkippedPacks += other.totalDmfileSkippedPacks
context.totalDmfileSkippedRows += other.totalDmfileSkippedRows
context.totalDmfileRoughSetIndexLoadTimeMs += other.totalDmfileRoughSetIndexLoadTimeMs
context.totalDmfileRoughSetIndexCheckTimeMs += other.totalDmfileRoughSetIndexCheckTimeMs
context.totalDmfileReadTimeMs += other.totalDmfileReadTimeMs
context.totalCreateSnapshotTimeMs += other.totalCreateSnapshotTimeMs
context.totalLocalRegionNum += other.totalLocalRegionNum
context.totalRemoteRegionNum += other.totalRemoteRegionNum
context.totalLearnerReadMs += other.totalLearnerReadMs
context.totalDisaggReadCacheHitSize += other.totalDisaggReadCacheHitSize
context.totalDisaggReadCacheMissSize += other.totalDisaggReadCacheMissSize
}

// Empty check whether TiFlashScanContext is Empty, if scan no pack and skip no pack, we regard it as empty
Expand Down
39 changes: 22 additions & 17 deletions util/execdetails/execdetails_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package execdetails

import (
"fmt"
"strconv"
"sync"
"testing"
Expand Down Expand Up @@ -135,18 +136,22 @@ func mockExecutorExecutionSummary(TimeProcessedNs, NumProducedRows, NumIteration
NumIterations: &NumIterations, XXX_unrecognized: nil}
}

func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexLoadTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs uint64, totalLocalRegionNum uint64, totalRemoteRegionNum uint64, ExecutorID string) *tipb.ExecutorExecutionSummary {
func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexCheckTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs, totalLocalRegionNum, totalRemoteRegionNum, totalLearnerReadMs, totalDisaggReadCacheHitSize, totalDisaggReadCacheMissSize uint64, ExecutorID string) *tipb.ExecutorExecutionSummary {
tiflashScanContext := tipb.TiFlashScanContext{
TotalDmfileScannedPacks: &totalDmfileScannedPacks,
TotalDmfileSkippedPacks: &totalDmfileSkippedPacks,
TotalDmfileScannedRows: &totalDmfileScannedRows,
TotalDmfileSkippedRows: &totalDmfileSkippedRows,
TotalDmfileRoughSetIndexLoadTimeMs: &totalDmfileRoughSetIndexLoadTimeMs,
TotalDmfileReadTimeMs: &totalDmfileReadTimeMs,
TotalCreateSnapshotTimeMs: &totalCreateSnapshotTimeMs,
TotalLocalRegionNum: &totalLocalRegionNum,
TotalRemoteRegionNum: &totalRemoteRegionNum,
TotalDmfileScannedPacks: &totalDmfileScannedPacks,
TotalDmfileSkippedPacks: &totalDmfileSkippedPacks,
TotalDmfileScannedRows: &totalDmfileScannedRows,
TotalDmfileSkippedRows: &totalDmfileSkippedRows,
TotalDmfileRoughSetIndexCheckTimeMs: &totalDmfileRoughSetIndexCheckTimeMs,
TotalDmfileReadTimeMs: &totalDmfileReadTimeMs,
TotalCreateSnapshotTimeMs: &totalCreateSnapshotTimeMs,
TotalLocalRegionNum: &totalLocalRegionNum,
TotalRemoteRegionNum: &totalRemoteRegionNum,
TotalLearnerReadMs: &totalLearnerReadMs,
TotalDisaggReadCacheHitSize: &totalDisaggReadCacheHitSize,
TotalDisaggReadCacheMissSize: &totalDisaggReadCacheMissSize,
}
fmt.Println("tiflashScanContext is ", tiflashScanContext.String())
return &tipb.ExecutorExecutionSummary{TimeProcessedNs: &TimeProcessedNs, NumProducedRows: &NumProducedRows,
NumIterations: &NumIterations, Concurrency: &Concurrency, ExecutorId: &ExecutorID, DetailInfo: &tipb.ExecutorExecutionSummary_TiflashScanContext{TiflashScanContext: &tiflashScanContext}, XXX_unrecognized: nil}
}
Expand Down Expand Up @@ -210,10 +215,10 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) {
tableScanID := 1
aggID := 2
tableReaderID := 3
stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, 10, 4, "tablescan_"+strconv.Itoa(tableScanID)))
stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID)))
stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, 5, 1, "aggregation_"+strconv.Itoa(aggID)))
stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, 1, 1, "aggregation_"+strconv.Itoa(aggID)))
stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, 10, 4, 1, 100, 50, "tablescan_"+strconv.Itoa(tableScanID)))
stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID)))
stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, 5, 1, 0, 20, 0, "aggregation_"+strconv.Itoa(aggID)))
stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, 1, 1, 0, 0, 0, "aggregation_"+strconv.Itoa(aggID)))
scanDetail := &util.ScanDetail{
TotalKeys: 10,
ProcessedKeys: 10,
Expand All @@ -227,15 +232,15 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) {
require.True(t, stats.ExistsCopStats(tableScanID))

cop := stats.GetOrCreateCopStats(tableScanID, "tiflash")
require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 202ms}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4}", cop.String())
require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_check_time: 15ms, total_read_time: 202ms, total_disagg_read_cache_hit_size: 100, total_disagg_read_cache_miss_size: 50}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4, total_learner_read_time: 1ms}", cop.String())

copStats := cop.stats["8.8.8.8"]
require.NotNil(t, copStats)

copStats.SetRowNum(10)
copStats.Record(time.Second, 10)
require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_load_time: 15ms, total_read_time: 200ms}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4}", copStats.String())
expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_load_time: 100ms, total_read_time: 3000ms}, total_create_snapshot_time: 50ms, total_local_region_num: 6, total_remote_region_num: 2}"
require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_check_time: 15ms, total_read_time: 200ms, total_disagg_read_cache_hit_size: 100, total_disagg_read_cache_miss_size: 50}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4, total_learner_read_time: 1ms}", copStats.String())
expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_check_time: 100ms, total_read_time: 3000ms, total_disagg_read_cache_hit_size: 20, total_disagg_read_cache_miss_size: 0}, total_create_snapshot_time: 50ms, total_local_region_num: 6, total_remote_region_num: 2, total_learner_read_time: 0ms}"
require.Equal(t, expected, stats.GetOrCreateCopStats(aggID, "tiflash").String())

rootStats := stats.GetRootStats(tableReaderID)
Expand Down