Skip to content

Commit

Permalink
*: Add more grafana charts for pd http api request (#33936)
Browse files Browse the repository at this point in the history 
ref #33916
  • Loading branch information
@lcwangchao
lcwangchao authored Apr 14, 2022
1 parent f988f54 commit 45e48b6
Show file tree
Hide file tree
Showing 9 changed files with 310 additions and 31 deletions.
9 changes: 6 additions & 3 deletions domain/infosync/info.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,7 @@ func GetTiFlashTableSyncProgress(ctx context.Context) (map[int64]float64, error)
return progressMap, nil
}

func doRequest(ctx context.Context, addrs []string, route, method string, body io.Reader) ([]byte, error) {
func doRequest(ctx context.Context, apiName string, addrs []string, route, method string, body io.Reader) ([]byte, error) {
var err error
var req *http.Request
var res *http.Response
Expand All @@ -391,8 +391,9 @@ func doRequest(ctx context.Context, addrs []string, route, method string, body i
}
start := time.Now()
res, err = doRequestWithFailpoint(req)
metrics.PDApiExecutionHistogram.WithLabelValues("placement").Observe(time.Since(start).Seconds())
if err == nil {
metrics.PDAPIExecutionHistogram.WithLabelValues(apiName).Observe(time.Since(start).Seconds())
metrics.PDAPIRequestCounter.WithLabelValues(apiName, res.Status).Inc()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
terror.Log(res.Body.Close())
Expand All @@ -415,8 +416,10 @@ func doRequest(ctx context.Context, addrs []string, route, method string, body i
terror.Log(res.Body.Close())
return bodyBytes, err
}
logutil.BgLogger().Warn("fail to doRequest, retry next address",
metrics.PDAPIRequestCounter.WithLabelValues(apiName, "network error").Inc()
logutil.BgLogger().Warn("fail to doRequest",
zap.Error(err),
zap.Bool("retry next address", idx == len(addrs)-1),
zap.String("method", method),
zap.String("hosts", addr),
zap.String("url", url),
Expand Down
8 changes: 4 additions & 4 deletions domain/infosync/label_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func (lm *PDLabelManager) PutLabelRule(ctx context.Context, rule *label.Rule) er
if err != nil {
return err
}
_, err = doRequest(ctx, lm.etcdCli.Endpoints(), path.Join(pdapi.Config, "region-label", "rule"), "POST", bytes.NewReader(r))
_, err = doRequest(ctx, "PutLabelRule", lm.etcdCli.Endpoints(), path.Join(pdapi.Config, "region-label", "rule"), "POST", bytes.NewReader(r))
return err
}

Expand All @@ -56,14 +56,14 @@ func (lm *PDLabelManager) UpdateLabelRules(ctx context.Context, patch *label.Rul
return err
}

_, err = doRequest(ctx, lm.etcdCli.Endpoints(), path.Join(pdapi.Config, "region-label", "rules"), "PATCH", bytes.NewReader(r))
_, err = doRequest(ctx, "UpdateLabelRules", lm.etcdCli.Endpoints(), path.Join(pdapi.Config, "region-label", "rules"), "PATCH", bytes.NewReader(r))
return err
}

// GetAllLabelRules implements GetAllLabelRules
func (lm *PDLabelManager) GetAllLabelRules(ctx context.Context) ([]*label.Rule, error) {
var rules []*label.Rule
res, err := doRequest(ctx, lm.etcdCli.Endpoints(), path.Join(pdapi.Config, "region-label", "rules"), "GET", nil)
res, err := doRequest(ctx, "GetAllLabelRules", lm.etcdCli.Endpoints(), path.Join(pdapi.Config, "region-label", "rules"), "GET", nil)

if err == nil && res != nil {
err = json.Unmarshal(res, &rules)
Expand All @@ -79,7 +79,7 @@ func (lm *PDLabelManager) GetLabelRules(ctx context.Context, ruleIDs []string) (
}

rules := []*label.Rule{}
res, err := doRequest(ctx, lm.etcdCli.Endpoints(), path.Join(pdapi.Config, "region-label", "rules", "ids"), "GET", bytes.NewReader(ids))
res, err := doRequest(ctx, "GetLabelRules", lm.etcdCli.Endpoints(), path.Join(pdapi.Config, "region-label", "rules", "ids"), "GET", bytes.NewReader(ids))

if err == nil && res != nil {
err = json.Unmarshal(res, &rules)
Expand Down
6 changes: 3 additions & 3 deletions domain/infosync/placement_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ type PDPlacementManager struct {
// GetRuleBundle is used to get one specific rule bundle from PD.
func (m *PDPlacementManager) GetRuleBundle(ctx context.Context, name string) (*placement.Bundle, error) {
bundle := &placement.Bundle{ID: name}
res, err := doRequest(ctx, m.etcdCli.Endpoints(), path.Join(pdapi.Config, "placement-rule", name), "GET", nil)
res, err := doRequest(ctx, "GetPlacementRule", m.etcdCli.Endpoints(), path.Join(pdapi.Config, "placement-rule", name), "GET", nil)
if err == nil && res != nil {
err = json.Unmarshal(res, bundle)
}
Expand All @@ -54,7 +54,7 @@ func (m *PDPlacementManager) GetRuleBundle(ctx context.Context, name string) (*p
// GetAllRuleBundles is used to get all rule bundles from PD. It is used to load full rules from PD while fullload infoschema.
func (m *PDPlacementManager) GetAllRuleBundles(ctx context.Context) ([]*placement.Bundle, error) {
var bundles []*placement.Bundle
res, err := doRequest(ctx, m.etcdCli.Endpoints(), path.Join(pdapi.Config, "placement-rule"), "GET", nil)
res, err := doRequest(ctx, "GetAllPlacementRules", m.etcdCli.Endpoints(), path.Join(pdapi.Config, "placement-rule"), "GET", nil)
if err == nil && res != nil {
err = json.Unmarshal(res, &bundles)
}
Expand All @@ -72,7 +72,7 @@ func (m *PDPlacementManager) PutRuleBundles(ctx context.Context, bundles []*plac
return err
}

_, err = doRequest(ctx, m.etcdCli.Endpoints(), path.Join(pdapi.Config, "placement-rule")+"?partial=true", "POST", bytes.NewReader(b))
_, err = doRequest(ctx, "PutPlacementRules", m.etcdCli.Endpoints(), path.Join(pdapi.Config, "placement-rule")+"?partial=true", "POST", bytes.NewReader(b))
return err
}

Expand Down
2 changes: 1 addition & 1 deletion domain/infosync/region.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func GetReplicationState(ctx context.Context, startKey []byte, endKey []byte) (P
return PlacementScheduleStatePending, errors.Errorf("pd unavailable")
}

res, err := doRequest(ctx, addrs, fmt.Sprintf("%s/replicated?startKey=%s&endKey=%s", pdapi.Regions, hex.EncodeToString(startKey), hex.EncodeToString(endKey)), "GET", nil)
res, err := doRequest(ctx, "GetReplicationState", addrs, fmt.Sprintf("%s/replicated?startKey=%s&endKey=%s", pdapi.Regions, hex.EncodeToString(startKey), hex.EncodeToString(endKey)), "GET", nil)
if err == nil && res != nil {
st := PlacementScheduleStatePending
// it should not fail
Expand Down
12 changes: 6 additions & 6 deletions domain/infosync/tiflash_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func (m *TiFlashPDPlacementManager) SetPlacementRule(ctx context.Context, rule p
}
j, _ := json.Marshal(rule)
buf := bytes.NewBuffer(j)
res, err := doRequest(ctx, m.etcdCli.Endpoints(), path.Join(pdapi.Config, "rule"), "POST", buf)
res, err := doRequest(ctx, "SetPlacementRule", m.etcdCli.Endpoints(), path.Join(pdapi.Config, "rule"), "POST", buf)
if err != nil {
return errors.Trace(err)
}
Expand All @@ -88,7 +88,7 @@ func (m *TiFlashPDPlacementManager) SetPlacementRule(ctx context.Context, rule p

// DeletePlacementRule is to delete placement rule for certain group.
func (m *TiFlashPDPlacementManager) DeletePlacementRule(ctx context.Context, group string, ruleID string) error {
res, err := doRequest(ctx, m.etcdCli.Endpoints(), path.Join(pdapi.Config, "rule", group, ruleID), "DELETE", nil)
res, err := doRequest(ctx, "DeletePlacementRule", m.etcdCli.Endpoints(), path.Join(pdapi.Config, "rule", group, ruleID), "DELETE", nil)
if err != nil {
return errors.Trace(err)
}
Expand All @@ -100,7 +100,7 @@ func (m *TiFlashPDPlacementManager) DeletePlacementRule(ctx context.Context, gro

// GetGroupRules to get all placement rule in a certain group.
func (m *TiFlashPDPlacementManager) GetGroupRules(ctx context.Context, group string) ([]placement.TiFlashRule, error) {
res, err := doRequest(ctx, m.etcdCli.Endpoints(), path.Join(pdapi.Config, "rules", "group", group), "GET", nil)
res, err := doRequest(ctx, "GetGroupRules", m.etcdCli.Endpoints(), path.Join(pdapi.Config, "rules", "group", group), "GET", nil)
if err != nil {
return nil, errors.Trace(err)
}
Expand Down Expand Up @@ -133,7 +133,7 @@ func (m *TiFlashPDPlacementManager) PostAccelerateSchedule(ctx context.Context,
return errors.Trace(err)
}
buf := bytes.NewBuffer(j)
res, err := doRequest(ctx, m.etcdCli.Endpoints(), "/pd/api/v1/regions/accelerate-schedule", "POST", buf)
res, err := doRequest(ctx, "PostAccelerateSchedule", m.etcdCli.Endpoints(), "/pd/api/v1/regions/accelerate-schedule", "POST", buf)
if err != nil {
return errors.Trace(err)
}
Expand All @@ -153,7 +153,7 @@ func (m *TiFlashPDPlacementManager) GetPDRegionRecordStats(ctx context.Context,
p := fmt.Sprintf("/pd/api/v1/stats/region?start_key=%s&end_key=%s",
url.QueryEscape(string(startKey)),
url.QueryEscape(string(endKey)))
res, err := doRequest(ctx, m.etcdCli.Endpoints(), p, "GET", nil)
res, err := doRequest(ctx, "GetPDRegionStats", m.etcdCli.Endpoints(), p, "GET", nil)
if err != nil {
return errors.Trace(err)
}
Expand All @@ -171,7 +171,7 @@ func (m *TiFlashPDPlacementManager) GetPDRegionRecordStats(ctx context.Context,
// GetStoresStat gets the TiKV store information by accessing PD's api.
func (m *TiFlashPDPlacementManager) GetStoresStat(ctx context.Context) (*helper.StoresStat, error) {
var storesStat helper.StoresStat
res, err := doRequest(ctx, m.etcdCli.Endpoints(), pdapi.Stores, "GET", nil)
res, err := doRequest(ctx, "GetStoresStat", m.etcdCli.Endpoints(), pdapi.Stores, "GET", nil)
if err != nil {
return nil, errors.Trace(err)
}
Expand Down
Loading

0 comments on commit 45e48b6

Please sign in to comment.