Skip to content

Commit

Permalink
util: Move tiflashScanContext outof basic stats (#56742)
Browse files Browse the repository at this point in the history 
close #56743
  • Loading branch information
@yibin87
yibin87 authored Oct 21, 2024
1 parent 64c8d31 commit dfd6cf2
Showing 1 changed file with 69 additions and 72 deletions.
141 changes: 69 additions & 72 deletions pkg/util/execdetails/execdetails.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,6 +553,8 @@ type basicCopRuntimeStats struct {
threads int32
totalTasks int32
procTimes Percentile[Duration]
// executor extra infos
tiflashScanContext TiFlashScanContext
}

type canGetFloat64 interface {
Expand Down Expand Up @@ -680,15 +682,15 @@ func (p *Percentile[valueType]) Sum() float64 {
// String implements the RuntimeStats interface.
func (e *basicCopRuntimeStats) String() string {
if e.storeType == "tiflash" {
return fmt.Sprintf("time:%v, loops:%d, threads:%d, ", FormatDuration(time.Duration(e.consume.Load())), e.loop.Load(), e.threads) + e.BasicRuntimeStats.tiflashScanContext.String()
return fmt.Sprintf("time:%v, loops:%d, threads:%d, ", FormatDuration(time.Duration(e.consume.Load())), e.loop.Load(), e.threads) + e.tiflashScanContext.String()
}
return fmt.Sprintf("time:%v, loops:%d", FormatDuration(time.Duration(e.consume.Load())), e.loop.Load())
}

// Clone implements the RuntimeStats interface.
func (e *basicCopRuntimeStats) Clone() RuntimeStats {
stats := &basicCopRuntimeStats{
BasicRuntimeStats: BasicRuntimeStats{tiflashScanContext: e.tiflashScanContext.Clone()},
BasicRuntimeStats: BasicRuntimeStats{},
threads: e.threads,
storeType: e.storeType,
totalTasks: e.totalTasks,
Expand All @@ -697,6 +699,7 @@ func (e *basicCopRuntimeStats) Clone() RuntimeStats {
stats.loop.Store(e.loop.Load())
stats.consume.Store(e.consume.Load())
stats.rows.Store(e.rows.Load())
stats.tiflashScanContext = e.tiflashScanContext.Clone()
return stats
}

Expand Down Expand Up @@ -750,54 +753,55 @@ func (crs *CopRuntimeStats) RecordOneCopTask(address string, summary *tipb.Execu
}
}
data := &basicCopRuntimeStats{
storeType: crs.storeType,
BasicRuntimeStats: BasicRuntimeStats{
tiflashScanContext: TiFlashScanContext{
dmfileDataScannedRows: summary.GetTiflashScanContext().GetDmfileDataScannedRows(),
dmfileDataSkippedRows: summary.GetTiflashScanContext().GetDmfileDataSkippedRows(),
dmfileMvccScannedRows: summary.GetTiflashScanContext().GetDmfileMvccScannedRows(),
dmfileMvccSkippedRows: summary.GetTiflashScanContext().GetDmfileMvccSkippedRows(),
dmfileLmFilterScannedRows: summary.GetTiflashScanContext().GetDmfileLmFilterScannedRows(),
dmfileLmFilterSkippedRows: summary.GetTiflashScanContext().GetDmfileLmFilterSkippedRows(),
totalDmfileRsCheckMs: summary.GetTiflashScanContext().GetTotalDmfileRsCheckMs(),
totalDmfileReadMs: summary.GetTiflashScanContext().GetTotalDmfileReadMs(),
totalBuildSnapshotMs: summary.GetTiflashScanContext().GetTotalBuildSnapshotMs(),
localRegions: summary.GetTiflashScanContext().GetLocalRegions(),
remoteRegions: summary.GetTiflashScanContext().GetRemoteRegions(),
totalLearnerReadMs: summary.GetTiflashScanContext().GetTotalLearnerReadMs(),
disaggReadCacheHitBytes: summary.GetTiflashScanContext().GetDisaggReadCacheHitBytes(),
disaggReadCacheMissBytes: summary.GetTiflashScanContext().GetDisaggReadCacheMissBytes(),
segments: summary.GetTiflashScanContext().GetSegments(),
readTasks: summary.GetTiflashScanContext().GetReadTasks(),
deltaRows: summary.GetTiflashScanContext().GetDeltaRows(),
deltaBytes: summary.GetTiflashScanContext().GetDeltaBytes(),
mvccInputRows: summary.GetTiflashScanContext().GetMvccInputRows(),
mvccInputBytes: summary.GetTiflashScanContext().GetMvccInputBytes(),
mvccOutputRows: summary.GetTiflashScanContext().GetMvccOutputRows(),
lmSkipRows: summary.GetTiflashScanContext().GetLmSkipRows(),
totalBuildBitmapMs: summary.GetTiflashScanContext().GetTotalBuildBitmapMs(),
totalBuildInputStreamMs: summary.GetTiflashScanContext().GetTotalBuildInputstreamMs(),
staleReadRegions: summary.GetTiflashScanContext().GetStaleReadRegions(),
minLocalStreamMs: summary.GetTiflashScanContext().GetMinLocalStreamMs(),
maxLocalStreamMs: summary.GetTiflashScanContext().GetMaxLocalStreamMs(),
minRemoteStreamMs: summary.GetTiflashScanContext().GetMinRemoteStreamMs(),
maxRemoteStreamMs: summary.GetTiflashScanContext().GetMaxRemoteStreamMs(),
regionsOfInstance: make(map[string]uint64),

totalVectorIdxLoadFromS3: summary.GetTiflashScanContext().GetTotalVectorIdxLoadFromS3(),
totalVectorIdxLoadFromDisk: summary.GetTiflashScanContext().GetTotalVectorIdxLoadFromDisk(),
totalVectorIdxLoadFromCache: summary.GetTiflashScanContext().GetTotalVectorIdxLoadFromCache(),
totalVectorIdxLoadTimeMs: summary.GetTiflashScanContext().GetTotalVectorIdxLoadTimeMs(),
totalVectorIdxSearchTimeMs: summary.GetTiflashScanContext().GetTotalVectorIdxSearchTimeMs(),
totalVectorIdxSearchVisitedNodes: summary.GetTiflashScanContext().GetTotalVectorIdxSearchVisitedNodes(),
totalVectorIdxSearchDiscardedNodes: summary.GetTiflashScanContext().GetTotalVectorIdxSearchDiscardedNodes(),
totalVectorIdxReadVecTimeMs: summary.GetTiflashScanContext().GetTotalVectorIdxReadVecTimeMs(),
totalVectorIdxReadOthersTimeMs: summary.GetTiflashScanContext().GetTotalVectorIdxReadOthersTimeMs(),
}}, threads: int32(summary.GetConcurrency()),
totalTasks: 1,
storeType: crs.storeType,
BasicRuntimeStats: BasicRuntimeStats{},
threads: int32(summary.GetConcurrency()),
totalTasks: 1,
tiflashScanContext: TiFlashScanContext{
dmfileDataScannedRows: summary.GetTiflashScanContext().GetDmfileDataScannedRows(),
dmfileDataSkippedRows: summary.GetTiflashScanContext().GetDmfileDataSkippedRows(),
dmfileMvccScannedRows: summary.GetTiflashScanContext().GetDmfileMvccScannedRows(),
dmfileMvccSkippedRows: summary.GetTiflashScanContext().GetDmfileMvccSkippedRows(),
dmfileLmFilterScannedRows: summary.GetTiflashScanContext().GetDmfileLmFilterScannedRows(),
dmfileLmFilterSkippedRows: summary.GetTiflashScanContext().GetDmfileLmFilterSkippedRows(),
totalDmfileRsCheckMs: summary.GetTiflashScanContext().GetTotalDmfileRsCheckMs(),
totalDmfileReadMs: summary.GetTiflashScanContext().GetTotalDmfileReadMs(),
totalBuildSnapshotMs: summary.GetTiflashScanContext().GetTotalBuildSnapshotMs(),
localRegions: summary.GetTiflashScanContext().GetLocalRegions(),
remoteRegions: summary.GetTiflashScanContext().GetRemoteRegions(),
totalLearnerReadMs: summary.GetTiflashScanContext().GetTotalLearnerReadMs(),
disaggReadCacheHitBytes: summary.GetTiflashScanContext().GetDisaggReadCacheHitBytes(),
disaggReadCacheMissBytes: summary.GetTiflashScanContext().GetDisaggReadCacheMissBytes(),
segments: summary.GetTiflashScanContext().GetSegments(),
readTasks: summary.GetTiflashScanContext().GetReadTasks(),
deltaRows: summary.GetTiflashScanContext().GetDeltaRows(),
deltaBytes: summary.GetTiflashScanContext().GetDeltaBytes(),
mvccInputRows: summary.GetTiflashScanContext().GetMvccInputRows(),
mvccInputBytes: summary.GetTiflashScanContext().GetMvccInputBytes(),
mvccOutputRows: summary.GetTiflashScanContext().GetMvccOutputRows(),
lmSkipRows: summary.GetTiflashScanContext().GetLmSkipRows(),
totalBuildBitmapMs: summary.GetTiflashScanContext().GetTotalBuildBitmapMs(),
totalBuildInputStreamMs: summary.GetTiflashScanContext().GetTotalBuildInputstreamMs(),
staleReadRegions: summary.GetTiflashScanContext().GetStaleReadRegions(),
minLocalStreamMs: summary.GetTiflashScanContext().GetMinLocalStreamMs(),
maxLocalStreamMs: summary.GetTiflashScanContext().GetMaxLocalStreamMs(),
minRemoteStreamMs: summary.GetTiflashScanContext().GetMinRemoteStreamMs(),
maxRemoteStreamMs: summary.GetTiflashScanContext().GetMaxRemoteStreamMs(),
regionsOfInstance: make(map[string]uint64),

totalVectorIdxLoadFromS3: summary.GetTiflashScanContext().GetTotalVectorIdxLoadFromS3(),
totalVectorIdxLoadFromDisk: summary.GetTiflashScanContext().GetTotalVectorIdxLoadFromDisk(),
totalVectorIdxLoadFromCache: summary.GetTiflashScanContext().GetTotalVectorIdxLoadFromCache(),
totalVectorIdxLoadTimeMs: summary.GetTiflashScanContext().GetTotalVectorIdxLoadTimeMs(),
totalVectorIdxSearchTimeMs: summary.GetTiflashScanContext().GetTotalVectorIdxSearchTimeMs(),
totalVectorIdxSearchVisitedNodes: summary.GetTiflashScanContext().GetTotalVectorIdxSearchVisitedNodes(),
totalVectorIdxSearchDiscardedNodes: summary.GetTiflashScanContext().GetTotalVectorIdxSearchDiscardedNodes(),
totalVectorIdxReadVecTimeMs: summary.GetTiflashScanContext().GetTotalVectorIdxReadVecTimeMs(),
totalVectorIdxReadOthersTimeMs: summary.GetTiflashScanContext().GetTotalVectorIdxReadOthersTimeMs(),
},
}
for _, instance := range summary.GetTiflashScanContext().GetRegionsOfInstance() {
data.BasicRuntimeStats.tiflashScanContext.regionsOfInstance[instance.GetInstanceId()] = instance.GetRegionNum()
data.tiflashScanContext.regionsOfInstance[instance.GetInstanceId()] = instance.GetRegionNum()
}
data.BasicRuntimeStats.loop.Store(int32(*summary.NumIterations))
data.BasicRuntimeStats.consume.Store(int64(*summary.TimeProcessedNs))
Expand Down Expand Up @@ -847,27 +851,25 @@ func (crs *CopRuntimeStats) String() string {
isTiFlashCop := crs.storeType == "tiflash"

buf := bytes.NewBuffer(make([]byte, 0, 16))
if totalTasks == 1 {
fmt.Fprintf(buf, "%v_task:{time:%v, loops:%d", crs.storeType, FormatDuration(time.Duration(procTimes.GetPercentile(0))), totalLoops)
if isTiFlashCop {
fmt.Fprintf(buf, ", threads:%d}", totalThreads)
if !totalTiFlashScanContext.Empty() {
buf.WriteString(", " + totalTiFlashScanContext.String())
{
printTiFlashScanContext := func() {
if isTiFlashCop {
fmt.Fprintf(buf, ", threads:%d}", totalThreads)
if !totalTiFlashScanContext.Empty() {
buf.WriteString(", " + totalTiFlashScanContext.String())
}
} else {
buf.WriteString("}")
}
} else {
buf.WriteString("}")
}
} else {
fmt.Fprintf(buf, "%v_task:{proc max:%v, min:%v, avg: %v, p80:%v, p95:%v, iters:%v, tasks:%v",
crs.storeType, FormatDuration(time.Duration(procTimes.GetMax().GetFloat64())), FormatDuration(time.Duration(procTimes.GetMin().GetFloat64())), FormatDuration(avgTime),
FormatDuration(time.Duration(procTimes.GetPercentile(0.8))), FormatDuration(time.Duration(procTimes.GetPercentile(0.95))), totalLoops, totalTasks)
if isTiFlashCop {
fmt.Fprintf(buf, ", threads:%d}", totalThreads)
if !totalTiFlashScanContext.Empty() {
buf.WriteString(", " + totalTiFlashScanContext.String())
}
if totalTasks == 1 {
fmt.Fprintf(buf, "%v_task:{time:%v, loops:%d", crs.storeType, FormatDuration(time.Duration(procTimes.GetPercentile(0))), totalLoops)
printTiFlashScanContext()
} else {
buf.WriteString("}")
fmt.Fprintf(buf, "%v_task:{proc max:%v, min:%v, avg: %v, p80:%v, p95:%v, iters:%v, tasks:%v",
crs.storeType, FormatDuration(time.Duration(procTimes.GetMax().GetFloat64())), FormatDuration(time.Duration(procTimes.GetMin().GetFloat64())), FormatDuration(avgTime),
FormatDuration(time.Duration(procTimes.GetPercentile(0.8))), FormatDuration(time.Duration(procTimes.GetPercentile(0.95))), totalLoops, totalTasks)
printTiFlashScanContext()
}
}
if !isTiFlashCop {
Expand Down Expand Up @@ -1217,8 +1219,6 @@ type BasicRuntimeStats struct {
consume atomic.Int64
// executor return row count.
rows atomic.Int64
// executor extra infos
tiflashScanContext TiFlashScanContext
}

// GetActRows return total rows of BasicRuntimeStats.
Expand All @@ -1228,9 +1228,7 @@ func (e *BasicRuntimeStats) GetActRows() int64 {

// Clone implements the RuntimeStats interface.
func (e *BasicRuntimeStats) Clone() RuntimeStats {
result := &BasicRuntimeStats{
tiflashScanContext: e.tiflashScanContext.Clone(),
}
result := &BasicRuntimeStats{}
result.loop.Store(e.loop.Load())
result.consume.Store(e.consume.Load())
result.rows.Store(e.rows.Load())
Expand All @@ -1246,7 +1244,6 @@ func (e *BasicRuntimeStats) Merge(rs RuntimeStats) {
e.loop.Add(tmp.loop.Load())
e.consume.Add(tmp.consume.Load())
e.rows.Add(tmp.rows.Load())
e.tiflashScanContext.Merge(tmp.tiflashScanContext)
}

// Tp implements the RuntimeStats interface.
Expand Down

0 comments on commit dfd6cf2

Please sign in to comment.