Skip to content

Commit

Permalink
Storages: Add Block pop-up latency metrics (#9260) (#9418)
Browse files Browse the repository at this point in the history 
ref #9417

Co-authored-by: jinhelin <[email protected]>
Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
Co-authored-by: JaySon <[email protected]>
  • Loading branch information
@ti-chi-bot @JinheLin
@ti-chi-bot @JaySon-Huang
4 people committed Sep 21, 2024
1 parent 8317761 commit a64d76b
Show file tree
Hide file tree
Showing 3 changed files with 150 additions and 5 deletions.
6 changes: 5 additions & 1 deletion dbms/src/Common/TiFlashMetrics.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -848,7 +848,11 @@ static_assert(RAFT_REGION_BIG_WRITE_THRES * 4 < RAFT_REGION_BIG_WRITE_MAX, "Inva
F(type_fg_read, {"type", "fg_read"}), \
F(type_bg_read, {"type", "bg_read"}), \
F(type_fg_write, {"type", "fg_write"}), \
F(type_bg_write, {"type", "bg_write"}))
F(type_bg_write, {"type", "bg_write"})) \
M(tiflash_read_thread_internal_us, \
"Durations of read thread internal components", \
Histogram, \
F(type_block_queue_pop_latency, {{"type", "block_queue_pop_latency"}}, ExpBuckets{1, 2, 20}))


/// Buckets with boundaries [start * base^0, start * base^1, ..., start * base^(size-1)]
Expand Down
21 changes: 17 additions & 4 deletions dbms/src/Storages/DeltaMerge/ReadThread/WorkQueue.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ class WorkQueue
std::condition_variable writer_cv;
std::condition_variable finish_cv;
PipeConditionVariable pipe_cv;
std::queue<T> queue;
// <value, push_timestamp_ns>
std::queue<std::pair<T, UInt64>> queue;
bool done;
std::size_t max_size;

Expand All @@ -49,6 +50,12 @@ class WorkQueue
return queue.size() >= max_size;
}

static void reportPopLatency(UInt64 push_timestamp_ns)
{
auto latency_ns = clock_gettime_ns_adjusted(push_timestamp_ns) - push_timestamp_ns;
GET_METRIC(tiflash_read_thread_internal_us, type_block_queue_pop_latency).Observe(latency_ns / 1000.0);
}

public:
/**
* Constructs an empty work queue with an optional max size.
Expand Down Expand Up @@ -99,7 +106,7 @@ class WorkQueue
{
return false;
}
queue.push(std::forward<U>(item));
queue.push(std::make_pair(std::forward<U>(item), clock_gettime_ns()));
peak_queue_size = std::max(queue.size(), peak_queue_size);
if (size != nullptr)
{
Expand All @@ -121,6 +128,7 @@ class WorkQueue
*/
bool pop(T & item)
{
UInt64 push_timestamp_ns = 0;
{
std::unique_lock<std::mutex> lock(mu);
++pop_times;
Expand All @@ -134,10 +142,12 @@ class WorkQueue
assert(done);
return false;
}
item = std::move(queue.front());
item = std::move(queue.front().first);
push_timestamp_ns = queue.front().second;
queue.pop();
}
writer_cv.notify_one();
reportPopLatency(push_timestamp_ns);
return true;
}

Expand All @@ -152,6 +162,7 @@ class WorkQueue
*/
bool tryPop(T & item)
{
UInt64 push_timestamp_ns = 0;
{
std::lock_guard lock(mu);
++pop_times;
Expand All @@ -167,10 +178,12 @@ class WorkQueue
return false;
}
}
item = std::move(queue.front());
item = std::move(queue.front().first);
push_timestamp_ns = queue.front().second;
queue.pop();
}
writer_cv.notify_one();
reportPopLatency(push_timestamp_ns);
return true;
}

Expand Down
128 changes: 128 additions & 0 deletions metrics/grafana/tiflash_summary.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8609,6 +8609,134 @@
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 165
},
"hiddenSeries": false,
"id": 292,
"legend": {
"alignAsTable": false,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.17",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "histogram_quantile(0.999, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "999-{{type}}",
"refId": "A"
},
{
"exemplar": true,
"expr": "histogram_quantile(0.99, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "99-{{type}}",
"refId": "B"
},
{
"exemplar": true,
"expr": "histogram_quantile(0.95, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "95-{{type}}",
"refId": "C"
},
{
"exemplar": true,
"expr": "histogram_quantile(0.80, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "80-{{type}}",
"refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Read Thread Internal Duration",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:222",
"format": "µs",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:223",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
Expand Down

0 comments on commit a64d76b

Please sign in to comment.