Skip to content

Commit

Permalink
api: use easyjson and context in regions interface (#6838)
Browse files Browse the repository at this point in the history 
close #6835

Signed-off-by: lhy1024 <[email protected]>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
  • Loading branch information
@lhy1024 @ti-chi-bot
lhy1024 and ti-chi-bot[bot] authored Jul 27, 2023
1 parent 7ac9e6b commit 4db1735
Show file tree
Hide file tree
Showing 4 changed files with 815 additions and 88 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ require (
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mailru/easyjson v0.7.6
github.com/mattn/go-colorable v0.1.8 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.8 // indirect
Expand Down
166 changes: 126 additions & 40 deletions server/api/region.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package api

import (
"container/heap"
"context"
"encoding/hex"
"fmt"
"net/http"
Expand All @@ -25,6 +26,7 @@ import (
"strings"

"github.com/gorilla/mux"
jwriter "github.com/mailru/easyjson/jwriter"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/kvproto/pkg/pdpb"
Expand Down Expand Up @@ -54,13 +56,34 @@ type MetaPeer struct {
IsLearner bool `json:"is_learner,omitempty"`
}

func (m *MetaPeer) setDefaultIfNil() {
if m.Peer == nil {
m.Peer = &metapb.Peer{
Id: m.GetId(),
StoreId: m.GetStoreId(),
Role: m.GetRole(),
IsWitness: m.GetIsWitness(),
}
}
}

// PDPeerStats is api compatible with *pdpb.PeerStats.
// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
type PDPeerStats struct {
*pdpb.PeerStats
Peer MetaPeer `json:"peer"`
}

func (s *PDPeerStats) setDefaultIfNil() {
if s.PeerStats == nil {
s.PeerStats = &pdpb.PeerStats{
Peer: s.GetPeer(),
DownSeconds: s.GetDownSeconds(),
}
}
s.Peer.setDefaultIfNil()
}

func fromPeer(peer *metapb.Peer) MetaPeer {
if peer == nil {
return MetaPeer{}
Expand Down Expand Up @@ -103,6 +126,7 @@ func fromPeerStatsSlice(peers []*pdpb.PeerStats) []PDPeerStats {

// RegionInfo records detail region info for api usage.
// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it.
// easyjson:json
type RegionInfo struct {
ID uint64 `json:"id"`
StartKey string `json:"start_key"`
Expand Down Expand Up @@ -169,9 +193,9 @@ func InitRegion(r *core.RegionInfo, s *RegionInfo) *RegionInfo {
s.ApproximateSize = r.GetApproximateSize()
s.ApproximateKeys = r.GetApproximateKeys()
s.ReplicationStatus = fromPBReplicationStatus(r.GetReplicationStatus())
s.Buckets = nil

keys := r.GetBuckets().GetKeys()

if len(keys) > 0 {
s.Buckets = make([]string, len(keys))
for i, key := range keys {
Expand Down Expand Up @@ -313,15 +337,48 @@ func newRegionsHandler(svr *server.Server, rd *render.Render) *regionsHandler {
}
}

func convertToAPIRegions(regions []*core.RegionInfo) *RegionsInfo {
regionInfos := make([]RegionInfo, len(regions))
// marshalRegionsInfoJSON marshals regions to bytes in `RegionsInfo`'s JSON format.
// It is used to reduce the cost of JSON serialization.
func marshalRegionsInfoJSON(ctx context.Context, regions []*core.RegionInfo) ([]byte, error) {
out := &jwriter.Writer{}
out.RawByte('{')

out.RawString("\"count\":")
out.Int(len(regions))

out.RawString(",\"regions\":")
out.RawByte('[')
region := &RegionInfo{}
for i, r := range regions {
InitRegion(r, &regionInfos[i])
}
return &RegionsInfo{
Count: len(regions),
Regions: regionInfos,
select {
case <-ctx.Done():
// Return early, avoid the unnecessary computation.
// See more details in https://github.com/tikv/pd/issues/6835
return nil, ctx.Err()
default:
}
if i > 0 {
out.RawByte(',')
}
InitRegion(r, region)
// EasyJSON will not check anonymous struct pointer field and will panic if the field is nil.
// So we need to set the field to default value explicitly when the anonymous struct pointer is nil.
region.Leader.setDefaultIfNil()
for i := range region.Peers {
region.Peers[i].setDefaultIfNil()
}
for i := range region.PendingPeers {
region.PendingPeers[i].setDefaultIfNil()
}
for i := range region.DownPeers {
region.DownPeers[i].setDefaultIfNil()
}
region.MarshalEasyJSON(out)
}
out.RawByte(']')

out.RawByte('}')
return out.Buffer.BuildBytes(), out.Error
}

// @Tags region
Expand All @@ -332,8 +389,12 @@ func convertToAPIRegions(regions []*core.RegionInfo) *RegionsInfo {
func (h *regionsHandler) GetRegions(w http.ResponseWriter, r *http.Request) {
rc := getCluster(r)
regions := rc.GetRegions()
regionsInfo := convertToAPIRegions(regions)
h.rd.JSON(w, http.StatusOK, regionsInfo)
b, err := marshalRegionsInfoJSON(r.Context(), regions)
if err != nil {
h.rd.JSON(w, http.StatusInternalServerError, err.Error())
return
}
h.rd.Data(w, http.StatusOK, b)
}

// @Tags region
Expand Down Expand Up @@ -363,8 +424,12 @@ func (h *regionsHandler) ScanRegions(w http.ResponseWriter, r *http.Request) {
limit = maxRegionLimit
}
regions := rc.ScanRegions([]byte(startKey), []byte(endKey), limit)
regionsInfo := convertToAPIRegions(regions)
h.rd.JSON(w, http.StatusOK, regionsInfo)
b, err := marshalRegionsInfoJSON(r.Context(), regions)
if err != nil {
h.rd.JSON(w, http.StatusInternalServerError, err.Error())
return
}
h.rd.Data(w, http.StatusOK, b)
}

// @Tags region
Expand Down Expand Up @@ -395,8 +460,12 @@ func (h *regionsHandler) GetStoreRegions(w http.ResponseWriter, r *http.Request)
return
}
regions := rc.GetStoreRegions(uint64(id))
regionsInfo := convertToAPIRegions(regions)
h.rd.JSON(w, http.StatusOK, regionsInfo)
b, err := marshalRegionsInfoJSON(r.Context(), regions)
if err != nil {
h.rd.JSON(w, http.StatusInternalServerError, err.Error())
return
}
h.rd.Data(w, http.StatusOK, b)
}

// @Tags region
Expand Down Expand Up @@ -445,8 +514,12 @@ func (h *regionsHandler) GetKeyspaceRegions(w http.ResponseWriter, r *http.Reque
txnRegion := rc.ScanRegions(regionBound.TxnLeftBound, regionBound.TxnRightBound, limit-len(regions))
regions = append(regions, txnRegion...)
}
regionsInfo := convertToAPIRegions(regions)
h.rd.JSON(w, http.StatusOK, regionsInfo)
b, err := marshalRegionsInfoJSON(r.Context(), regions)
if err != nil {
h.rd.JSON(w, http.StatusInternalServerError, err.Error())
return
}
h.rd.Data(w, http.StatusOK, b)
}

// @Tags region
Expand All @@ -455,22 +528,27 @@ func (h *regionsHandler) GetKeyspaceRegions(w http.ResponseWriter, r *http.Reque
// @Success 200 {object} RegionsInfo
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /regions/check/miss-peer [get]
func (h *regionsHandler) GetMissPeerRegions(w http.ResponseWriter, _ *http.Request) {
h.getRegionsByType(w, statistics.MissPeer)
func (h *regionsHandler) GetMissPeerRegions(w http.ResponseWriter, r *http.Request) {
h.getRegionsByType(w, statistics.MissPeer, r)
}

func (h *regionsHandler) getRegionsByType(
w http.ResponseWriter,
typ statistics.RegionStatisticType,
r *http.Request,
) {
handler := h.svr.GetHandler()
regions, err := handler.GetRegionsByType(typ)
if err != nil {
h.rd.JSON(w, http.StatusInternalServerError, err.Error())
return
}
regionsInfo := convertToAPIRegions(regions)
h.rd.JSON(w, http.StatusOK, regionsInfo)
b, err := marshalRegionsInfoJSON(r.Context(), regions)
if err != nil {
h.rd.JSON(w, http.StatusInternalServerError, err.Error())
return
}
h.rd.Data(w, http.StatusOK, b)
}

// @Tags region
Expand All @@ -479,8 +557,8 @@ func (h *regionsHandler) getRegionsByType(
// @Success 200 {object} RegionsInfo
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /regions/check/extra-peer [get]
func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, _ *http.Request) {
h.getRegionsByType(w, statistics.ExtraPeer)
func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, r *http.Request) {
h.getRegionsByType(w, statistics.ExtraPeer, r)
}

// @Tags region
Expand All @@ -489,8 +567,8 @@ func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, _ *http.Requ
// @Success 200 {object} RegionsInfo
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /regions/check/pending-peer [get]
func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, _ *http.Request) {
h.getRegionsByType(w, statistics.PendingPeer)
func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, r *http.Request) {
h.getRegionsByType(w, statistics.PendingPeer, r)
}

// @Tags region
Expand All @@ -499,8 +577,8 @@ func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, _ *http.Re
// @Success 200 {object} RegionsInfo
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /regions/check/down-peer [get]
func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, _ *http.Request) {
h.getRegionsByType(w, statistics.DownPeer)
func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, r *http.Request) {
h.getRegionsByType(w, statistics.DownPeer, r)
}

// @Tags region
Expand All @@ -509,8 +587,8 @@ func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, _ *http.Reque
// @Success 200 {object} RegionsInfo
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /regions/check/learner-peer [get]
func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, _ *http.Request) {
h.getRegionsByType(w, statistics.LearnerPeer)
func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, r *http.Request) {
h.getRegionsByType(w, statistics.LearnerPeer, r)
}

// @Tags region
Expand All @@ -519,8 +597,8 @@ func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, _ *http.Re
// @Success 200 {object} RegionsInfo
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /regions/check/offline-peer [get]
func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, _ *http.Request) {
h.getRegionsByType(w, statistics.OfflinePeer)
func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, r *http.Request) {
h.getRegionsByType(w, statistics.OfflinePeer, r)
}

// @Tags region
Expand All @@ -529,8 +607,8 @@ func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, _ *http.Re
// @Success 200 {object} RegionsInfo
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /regions/check/oversized-region [get]
func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, _ *http.Request) {
h.getRegionsByType(w, statistics.OversizedRegion)
func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, r *http.Request) {
h.getRegionsByType(w, statistics.OversizedRegion, r)
}

// @Tags region
Expand All @@ -539,8 +617,8 @@ func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, _ *http.Requ
// @Success 200 {object} RegionsInfo
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /regions/check/undersized-region [get]
func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, _ *http.Request) {
h.getRegionsByType(w, statistics.UndersizedRegion)
func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, r *http.Request) {
h.getRegionsByType(w, statistics.UndersizedRegion, r)
}

// @Tags region
Expand All @@ -549,8 +627,8 @@ func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, _ *http.Req
// @Success 200 {object} RegionsInfo
// @Failure 500 {string} string "PD server failed to proceed the request."
// @Router /regions/check/empty-region [get]
func (h *regionsHandler) GetEmptyRegions(w http.ResponseWriter, _ *http.Request) {
h.getRegionsByType(w, statistics.EmptyRegion)
func (h *regionsHandler) GetEmptyRegions(w http.ResponseWriter, r *http.Request) {
h.getRegionsByType(w, statistics.EmptyRegion, r)
}

type histItem struct {
Expand Down Expand Up @@ -690,8 +768,12 @@ func (h *regionsHandler) GetRegionSiblings(w http.ResponseWriter, r *http.Reques
}

left, right := rc.GetAdjacentRegions(region)
regionsInfo := convertToAPIRegions([]*core.RegionInfo{left, right})
h.rd.JSON(w, http.StatusOK, regionsInfo)
b, err := marshalRegionsInfoJSON(r.Context(), []*core.RegionInfo{left, right})
if err != nil {
h.rd.JSON(w, http.StatusInternalServerError, err.Error())
return
}
h.rd.Data(w, http.StatusOK, b)
}

const (
Expand Down Expand Up @@ -909,8 +991,12 @@ func (h *regionsHandler) GetTopNRegions(w http.ResponseWriter, r *http.Request,
limit = maxRegionLimit
}
regions := TopNRegions(rc.GetRegions(), less, limit)
regionsInfo := convertToAPIRegions(regions)
h.rd.JSON(w, http.StatusOK, regionsInfo)
b, err := marshalRegionsInfoJSON(r.Context(), regions)
if err != nil {
h.rd.JSON(w, http.StatusInternalServerError, err.Error())
return
}
h.rd.Data(w, http.StatusOK, b)
}

// @Tags region
Expand Down
Loading

0 comments on commit 4db1735

Please sign in to comment.