From 2f94024b356f774d0032ad069babb1a3fce6cfbd Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 15:21:24 +0530 Subject: [PATCH 01/24] init --- api/operator_raft.go | 17 +++++ api/operator_raft_test.go | 19 +++++ .../raft/listpeers/operator_raft_list.go | 71 +++++++++++++++++-- .../raft/listpeers/operator_raft_list_test.go | 27 +++++++ 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/api/operator_raft.go b/api/operator_raft.go index 393d6fb3c5f7..6ee75c4d8bb0 100644 --- a/api/operator_raft.go +++ b/api/operator_raft.go @@ -122,3 +122,20 @@ func (op *Operator) RaftRemovePeerByID(id string, q *WriteOptions) error { } return nil } + +// GetAutoPilotHealth is used to query the autopilot health. +func (op *Operator) GetAutoPilotHealth(q *QueryOptions) (*OperatorHealthReply, error) { + r := op.c.newRequest("GET", "/v1/operator/autopilot/health") + r.setQueryOptions(q) + _, resp, err := op.c.doRequest(r) + if err != nil { + return nil, err + } + defer closeResponseBody(resp) + + var out OperatorHealthReply + if err := decodeBody(resp, &out); err != nil { + return nil, err + } + return &out, nil +} diff --git a/api/operator_raft_test.go b/api/operator_raft_test.go index 6e3b7fc0e310..57b50b745269 100644 --- a/api/operator_raft_test.go +++ b/api/operator_raft_test.go @@ -57,3 +57,22 @@ func TestAPI_OperatorRaftLeaderTransfer(t *testing.T) { t.Fatalf("err:%v", transfer) } } + +func TestAPI_GetAutoPilotHealth(t *testing.T) { + t.Parallel() + c, s := makeClient(t) + defer s.Stop() + + operator := c.Operator() + out, err := operator.GetAutoPilotHealth(nil) + if err != nil { + t.Fatalf("err: %v", err) + } + + if len(out.Servers) != 1 || + !out.Servers[0].Leader || + !out.Servers[0].Voter || + out.Servers[0].LastIndex <= 0 { + t.Fatalf("bad: %v", out) + } +} diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 47bd161fed48..9cc4db67250a 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -24,10 +24,16 @@ type cmd struct { flags *flag.FlagSet http *flags.HTTPFlags help string + + // flags + detailed bool } func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) + c.flags.BoolVar(&c.detailed, "detailed", false, + "Outputs additional information 'commit_index' which is "+ + "the index of the server's last committed Raft log entry.") c.http = &flags.HTTPFlags{} flags.Merge(c.flags, c.http.ClientFlags()) flags.Merge(c.flags, c.http.ServerFlags()) @@ -51,13 +57,22 @@ func (c *cmd) Run(args []string) int { } // Fetch the current configuration. - result, err := raftListPeers(client, c.http.Stale()) - if err != nil { - c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) - return 1 + if c.detailed { + result, err := raftListPeersDetailed(client, c.http.Stale()) + if err != nil { + c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) + return 1 + } + c.UI.Output(result) + } else { + result, err := raftListPeers(client, c.http.Stale()) + if err != nil { + c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) + return 1 + } + c.UI.Output(result) } - c.UI.Output(result) return 0 } @@ -89,6 +104,52 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil } +func raftListPeersDetailed(client *api.Client, stale bool) (string, error) { + q := &api.QueryOptions{ + AllowStale: stale, + } + reply, err := client.Operator().RaftGetConfiguration(q) + if err != nil { + return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err) + } + + autoPilotReply, err := client.Operator().GetAutoPilotHealth(q) + if err != nil { + return "", fmt.Errorf("Failed to retrieve autopilot health: %v", err) + } + + serverHealthDataMap := make(map[string]api.ServerHealth) + + for _, serverHealthData := range autoPilotReply.Servers { + serverHealthDataMap[serverHealthData.ID] = serverHealthData + } + + // Format it as a nice table. + result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol\x1fCommitIndex"} + for _, s := range reply.Servers { + raftProtocol := s.ProtocolVersion + + if raftProtocol == "" { + raftProtocol = "<=1" + } + state := "follower" + if s.Leader { + state = "leader" + } + + serverHealthData, ok := serverHealthDataMap[s.ID] + if ok { + result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverHealthData.LastIndex)) + } else { + result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "")) + } + } + + return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil +} + func (c *cmd) Synopsis() string { return synopsis } diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index 15bd1bfbe34f..a610c099955f 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -46,6 +46,33 @@ func TestOperatorRaftListPeersCommand(t *testing.T) { } } +func TestOperatorRaftListPeersCommandDetailed(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + + t.Parallel() + a := agent.NewTestAgent(t, ``) + defer a.Shutdown() + + expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3", + a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) + + // Test the list-peers subcommand directly + ui := cli.NewMockUi() + c := New(ui) + args := []string{"-http-addr=" + a.HTTPAddr(), "-detailed"} + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + output := strings.TrimSpace(ui.OutputWriter.String()) + if !strings.Contains(output, expected) { + t.Fatalf("bad: %q, %q", output, expected) + } +} + func TestOperatorRaftListPeersCommand_verticalBar(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") From 7626d0992dd59f424d893dd462bbde3a9cb0646c Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 15:26:50 +0530 Subject: [PATCH 02/24] fix tests --- command/operator/raft/listpeers/operator_raft_list_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index a610c099955f..f60cad21e279 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -55,7 +55,7 @@ func TestOperatorRaftListPeersCommandDetailed(t *testing.T) { a := agent.NewTestAgent(t, ``) defer a.Shutdown() - expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3", + expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1", a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) // Test the list-peers subcommand directly From 79aabc958aed60b2bd97c836529d53a627262124 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 16:31:54 +0530 Subject: [PATCH 03/24] added -detailed in docs --- website/content/commands/operator/raft.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 857a9ee1ac12..7c948c1e14e3 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -73,6 +73,10 @@ configuration. we recommend setting this option to `true. Default is `false`. +- `-detailed` - Outputs additional information 'commit_index' which is +the index of the server's last committed Raft log entry. +Default is `false`. + ## remove-peer Corresponding HTTP API Endpoint: [\[DELETE\] /v1/operator/raft/peer](/consul/api-docs/operator/raft#delete-raft-peer) From 44eee4168b5a0d405636dbcb5b14ee7959334550 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 16:44:01 +0530 Subject: [PATCH 04/24] added change log --- .changelog/17582.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/17582.txt diff --git a/.changelog/17582.txt b/.changelog/17582.txt new file mode 100644 index 000000000000..3beae9d726a0 --- /dev/null +++ b/.changelog/17582.txt @@ -0,0 +1,3 @@ +```release-note:feature +improved consul operator raft list-peers command added -detailed flag which will print CommitIndex in the table in response +``` From d118518e3219fce61fa0fd58797d6dd8b57bf89c Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Tue, 6 Jun 2023 17:23:33 +0530 Subject: [PATCH 05/24] fix doc --- website/content/commands/operator/raft.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 7c948c1e14e3..76f9be838b54 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -70,7 +70,7 @@ configuration. - `-stale` - Enables non-leader servers to provide cluster state information. If the cluster is in an outage state without a leader, - we recommend setting this option to `true. + we recommend setting this option to `true`. Default is `false`. - `-detailed` - Outputs additional information 'commit_index' which is From d46a0744c6017567b5f19db2be34198f9202afeb Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Wed, 7 Jun 2023 10:14:12 +0530 Subject: [PATCH 06/24] checking for entry in map --- .../raft/listpeers/operator_raft_list.go | 73 +++++++------------ .../raft/listpeers/operator_raft_list_test.go | 29 +------- website/content/commands/operator/raft.mdx | 4 - 3 files changed, 27 insertions(+), 79 deletions(-) diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 9cc4db67250a..4e841dcfaa8e 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -56,22 +56,12 @@ func (c *cmd) Run(args []string) int { return 1 } - // Fetch the current configuration. - if c.detailed { - result, err := raftListPeersDetailed(client, c.http.Stale()) - if err != nil { - c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) - return 1 - } - c.UI.Output(result) - } else { - result, err := raftListPeers(client, c.http.Stale()) - if err != nil { - c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) - return 1 - } - c.UI.Output(result) + result, err := raftListPeers(client, c.http.Stale()) + if err != nil { + c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) + return 1 } + c.UI.Output(result) return 0 } @@ -85,47 +75,29 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err) } - // Format it as a nice table. - result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol"} - for _, s := range reply.Servers { - raftProtocol := s.ProtocolVersion - - if raftProtocol == "" { - raftProtocol = "<=1" - } - state := "follower" - if s.Leader { - state = "leader" - } - result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol)) - } - - return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil -} - -func raftListPeersDetailed(client *api.Client, stale bool) (string, error) { - q := &api.QueryOptions{ - AllowStale: stale, - } - reply, err := client.Operator().RaftGetConfiguration(q) - if err != nil { - return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err) - } - autoPilotReply, err := client.Operator().GetAutoPilotHealth(q) if err != nil { return "", fmt.Errorf("Failed to retrieve autopilot health: %v", err) } serverHealthDataMap := make(map[string]api.ServerHealth) + leaderLastCommitIndex := uint64(0) for _, serverHealthData := range autoPilotReply.Servers { serverHealthDataMap[serverHealthData.ID] = serverHealthData } + for _, s := range reply.Servers { + if s.Leader { + serverHealthDataLeader, ok := serverHealthDataMap[s.ID] + if ok { + leaderLastCommitIndex = serverHealthDataLeader.LastIndex + } + } + } + // Format it as a nice table. - result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol\x1fCommitIndex"} + result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol\x1fCommit Index\x1fTrails Leader By"} for _, s := range reply.Servers { raftProtocol := s.ProtocolVersion @@ -139,11 +111,18 @@ func raftListPeersDetailed(client *api.Client, stale bool) (string, error) { serverHealthData, ok := serverHealthDataMap[s.ID] if ok { - result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverHealthData.LastIndex)) + trailsLeaderBy := leaderLastCommitIndex - serverHealthData.LastIndex + trailsLeaderByText := fmt.Sprintf("%d Commits", trailsLeaderBy) + if s.Leader { + trailsLeaderByText = "_" + } else if trailsLeaderBy <= 1 { + trailsLeaderByText = fmt.Sprintf("%d Commit", trailsLeaderBy) + } + result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v\x1f%s", + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverHealthData.LastIndex, trailsLeaderByText)) } else { result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "")) + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "_")) } } diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index f60cad21e279..f6db9a3d0749 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -28,34 +28,7 @@ func TestOperatorRaftListPeersCommand(t *testing.T) { a := agent.NewTestAgent(t, ``) defer a.Shutdown() - expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3", - a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) - - // Test the list-peers subcommand directly - ui := cli.NewMockUi() - c := New(ui) - args := []string{"-http-addr=" + a.HTTPAddr()} - - code := c.Run(args) - if code != 0 { - t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) - } - output := strings.TrimSpace(ui.OutputWriter.String()) - if !strings.Contains(output, expected) { - t.Fatalf("bad: %q, %q", output, expected) - } -} - -func TestOperatorRaftListPeersCommandDetailed(t *testing.T) { - if testing.Short() { - t.Skip("too slow for testing.Short") - } - - t.Parallel() - a := agent.NewTestAgent(t, ``) - defer a.Shutdown() - - expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1", + expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1 _", a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) // Test the list-peers subcommand directly diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 76f9be838b54..e5bf6d1e0c81 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -73,10 +73,6 @@ configuration. we recommend setting this option to `true`. Default is `false`. -- `-detailed` - Outputs additional information 'commit_index' which is -the index of the server's last committed Raft log entry. -Default is `false`. - ## remove-peer Corresponding HTTP API Endpoint: [\[DELETE\] /v1/operator/raft/peer](/consul/api-docs/operator/raft#delete-raft-peer) From 539dbc74da822203b196cfe28e376d1e0d08097b Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Wed, 7 Jun 2023 11:11:46 +0530 Subject: [PATCH 07/24] fix tests --- command/operator/raft/listpeers/operator_raft_list_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index f6db9a3d0749..d3ab98df0ee8 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -34,7 +34,7 @@ func TestOperatorRaftListPeersCommand(t *testing.T) { // Test the list-peers subcommand directly ui := cli.NewMockUi() c := New(ui) - args := []string{"-http-addr=" + a.HTTPAddr(), "-detailed"} + args := []string{"-http-addr=" + a.HTTPAddr()} code := c.Run(args) if code != 0 { From 604b6161c5cb2e994c77aa13b748694cb12333bd Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Wed, 7 Jun 2023 11:12:53 +0530 Subject: [PATCH 08/24] removed detailed flag --- command/operator/raft/listpeers/operator_raft_list.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 4e841dcfaa8e..dfe8586a8759 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -24,9 +24,6 @@ type cmd struct { flags *flag.FlagSet http *flags.HTTPFlags help string - - // flags - detailed bool } func (c *cmd) init() { From 3a63b3d73961caf8142f7d87dedf11479fc306af Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Wed, 7 Jun 2023 11:13:13 +0530 Subject: [PATCH 09/24] removed detailed flag --- command/operator/raft/listpeers/operator_raft_list.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index dfe8586a8759..7acdac521cfa 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -28,9 +28,6 @@ type cmd struct { func (c *cmd) init() { c.flags = flag.NewFlagSet("", flag.ContinueOnError) - c.flags.BoolVar(&c.detailed, "detailed", false, - "Outputs additional information 'commit_index' which is "+ - "the index of the server's last committed Raft log entry.") c.http = &flags.HTTPFlags{} flags.Merge(c.flags, c.http.ClientFlags()) flags.Merge(c.flags, c.http.ServerFlags()) From 008946ca93f1156150f8b0ac2ad779df4fdf2428 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Wed, 7 Jun 2023 11:20:03 +0530 Subject: [PATCH 10/24] revert unwanted changes --- command/operator/raft/listpeers/operator_raft_list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 7acdac521cfa..de34f0cb57b7 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -50,13 +50,13 @@ func (c *cmd) Run(args []string) int { return 1 } + // Fetch the current configuration. result, err := raftListPeers(client, c.http.Stale()) if err != nil { c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) return 1 } c.UI.Output(result) - return 0 } From 509062e0acf80ef981ffc44ca8f00969f9250edd Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Wed, 7 Jun 2023 11:23:45 +0530 Subject: [PATCH 11/24] removed unwanted changes --- command/operator/raft/listpeers/operator_raft_list.go | 1 + 1 file changed, 1 insertion(+) diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index de34f0cb57b7..23f12bf227c7 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -56,6 +56,7 @@ func (c *cmd) Run(args []string) int { c.UI.Error(fmt.Sprintf("Error getting peers: %v", err)) return 1 } + c.UI.Output(result) return 0 } From cf22ea5325aebfae94a87cde9f503bbe76e534ff Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Wed, 7 Jun 2023 11:26:33 +0530 Subject: [PATCH 12/24] updated change log --- .changelog/17582.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.changelog/17582.txt b/.changelog/17582.txt index 3beae9d726a0..8f29e167f028 100644 --- a/.changelog/17582.txt +++ b/.changelog/17582.txt @@ -1,3 +1,5 @@ ```release-note:feature -improved consul operator raft list-peers command added -detailed flag which will print CommitIndex in the table in response +improved consul operator raft list-peers command added CommitIndex as well as Trails Leader By in the output table +Commit Index is the LastIndex from /v1/operator/autopilot/health and Trails Leader By is the difference from leader +last index and followers last index ``` From 5c2a58799340d4e1b435dcf01abc9e3d17e2ad27 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Thu, 8 Jun 2023 21:05:57 +0530 Subject: [PATCH 13/24] pr review comment changes --- .changelog/17582.txt | 4 +--- api/operator_raft.go | 17 ----------------- api/operator_raft_test.go | 19 ------------------- .../raft/listpeers/operator_raft_list.go | 10 +++++----- .../raft/listpeers/operator_raft_list_test.go | 2 +- 5 files changed, 7 insertions(+), 45 deletions(-) diff --git a/.changelog/17582.txt b/.changelog/17582.txt index 8f29e167f028..dce53703370f 100644 --- a/.changelog/17582.txt +++ b/.changelog/17582.txt @@ -1,5 +1,3 @@ ```release-note:feature -improved consul operator raft list-peers command added CommitIndex as well as Trails Leader By in the output table -Commit Index is the LastIndex from /v1/operator/autopilot/health and Trails Leader By is the difference from leader -last index and followers last index +raft list-peers shows the number of commits each follower is trailing the leader by to aid in troubleshooting. ``` diff --git a/api/operator_raft.go b/api/operator_raft.go index 6ee75c4d8bb0..393d6fb3c5f7 100644 --- a/api/operator_raft.go +++ b/api/operator_raft.go @@ -122,20 +122,3 @@ func (op *Operator) RaftRemovePeerByID(id string, q *WriteOptions) error { } return nil } - -// GetAutoPilotHealth is used to query the autopilot health. -func (op *Operator) GetAutoPilotHealth(q *QueryOptions) (*OperatorHealthReply, error) { - r := op.c.newRequest("GET", "/v1/operator/autopilot/health") - r.setQueryOptions(q) - _, resp, err := op.c.doRequest(r) - if err != nil { - return nil, err - } - defer closeResponseBody(resp) - - var out OperatorHealthReply - if err := decodeBody(resp, &out); err != nil { - return nil, err - } - return &out, nil -} diff --git a/api/operator_raft_test.go b/api/operator_raft_test.go index 57b50b745269..6e3b7fc0e310 100644 --- a/api/operator_raft_test.go +++ b/api/operator_raft_test.go @@ -57,22 +57,3 @@ func TestAPI_OperatorRaftLeaderTransfer(t *testing.T) { t.Fatalf("err:%v", transfer) } } - -func TestAPI_GetAutoPilotHealth(t *testing.T) { - t.Parallel() - c, s := makeClient(t) - defer s.Stop() - - operator := c.Operator() - out, err := operator.GetAutoPilotHealth(nil) - if err != nil { - t.Fatalf("err: %v", err) - } - - if len(out.Servers) != 1 || - !out.Servers[0].Leader || - !out.Servers[0].Voter || - out.Servers[0].LastIndex <= 0 { - t.Fatalf("bad: %v", out) - } -} diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 23f12bf227c7..10c9c57b5a63 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -70,7 +70,7 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err) } - autoPilotReply, err := client.Operator().GetAutoPilotHealth(q) + autoPilotReply, err := client.Operator().AutopilotServerHealth(q) if err != nil { return "", fmt.Errorf("Failed to retrieve autopilot health: %v", err) } @@ -107,17 +107,17 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { serverHealthData, ok := serverHealthDataMap[s.ID] if ok { trailsLeaderBy := leaderLastCommitIndex - serverHealthData.LastIndex - trailsLeaderByText := fmt.Sprintf("%d Commits", trailsLeaderBy) + trailsLeaderByText := fmt.Sprintf("%d commits", trailsLeaderBy) if s.Leader { - trailsLeaderByText = "_" + trailsLeaderByText = "-" } else if trailsLeaderBy <= 1 { - trailsLeaderByText = fmt.Sprintf("%d Commit", trailsLeaderBy) + trailsLeaderByText = fmt.Sprintf("%d commit", trailsLeaderBy) } result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v\x1f%s", s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverHealthData.LastIndex, trailsLeaderByText)) } else { result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "_")) + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "-")) } } diff --git a/command/operator/raft/listpeers/operator_raft_list_test.go b/command/operator/raft/listpeers/operator_raft_list_test.go index d3ab98df0ee8..0694e0dd1046 100644 --- a/command/operator/raft/listpeers/operator_raft_list_test.go +++ b/command/operator/raft/listpeers/operator_raft_list_test.go @@ -28,7 +28,7 @@ func TestOperatorRaftListPeersCommand(t *testing.T) { a := agent.NewTestAgent(t, ``) defer a.Shutdown() - expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1 _", + expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1 -", a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort) // Test the list-peers subcommand directly From 8e1f7285d00d968130d98f2d857e7839c5cf2013 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Thu, 8 Jun 2023 22:07:18 +0530 Subject: [PATCH 14/24] pr comment changes single API instead of two --- agent/consul/operator_raft_endpoint.go | 7 +++++++ agent/structs/operator.go | 3 +++ api/operator_raft.go | 3 +++ .../raft/listpeers/operator_raft_list.go | 21 +++++++------------ 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/agent/consul/operator_raft_endpoint.go b/agent/consul/operator_raft_endpoint.go index f619b611f7da..7b0bcbc5cc03 100644 --- a/agent/consul/operator_raft_endpoint.go +++ b/agent/consul/operator_raft_endpoint.go @@ -48,6 +48,12 @@ func (op *Operator) RaftGetConfiguration(args *structs.DCSpecificRequest, reply serverMap[raft.ServerAddress(addr)] = member } + serverIDLastIndexMap := make(map[raft.ServerID]uint64) + + for _, serverState := range op.srv.autopilot.GetState().Servers { + serverIDLastIndexMap[serverState.Server.ID] = serverState.Stats.LastIndex + } + // Fill out the reply. leader := op.srv.raft.Leader() reply.Index = future.Index() @@ -66,6 +72,7 @@ func (op *Operator) RaftGetConfiguration(args *structs.DCSpecificRequest, reply Leader: server.Address == leader, Voter: server.Suffrage == raft.Voter, ProtocolVersion: raftProtocolVersion, + LastIndex: serverIDLastIndexMap[server.ID], } reply.Servers = append(reply.Servers, entry) } diff --git a/agent/structs/operator.go b/agent/structs/operator.go index f5c2b8ac86df..05862861e3f8 100644 --- a/agent/structs/operator.go +++ b/agent/structs/operator.go @@ -34,6 +34,9 @@ type RaftServer struct { // it's a non-voting server, which will be added in a future release of // Consul. Voter bool + + // LastIndex is the last log index this server has a record of in its Raft log. + LastIndex uint64 } // RaftConfigurationResponse is returned when querying for the current Raft diff --git a/api/operator_raft.go b/api/operator_raft.go index 393d6fb3c5f7..d72c00c97b93 100644 --- a/api/operator_raft.go +++ b/api/operator_raft.go @@ -28,6 +28,9 @@ type RaftServer struct { // it's a non-voting server, which will be added in a future release of // Consul. Voter bool + + // LastIndex is the last log index this server has a record of in its Raft log. + LastIndex uint64 } // RaftConfiguration is returned when querying for the current Raft configuration. diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 10c9c57b5a63..495b2f64d185 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -70,23 +70,18 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err) } - autoPilotReply, err := client.Operator().AutopilotServerHealth(q) - if err != nil { - return "", fmt.Errorf("Failed to retrieve autopilot health: %v", err) - } - - serverHealthDataMap := make(map[string]api.ServerHealth) leaderLastCommitIndex := uint64(0) + serverIdLastIndexMap := make(map[string]uint64) - for _, serverHealthData := range autoPilotReply.Servers { - serverHealthDataMap[serverHealthData.ID] = serverHealthData + for _, raftServer := range reply.Servers { + serverIdLastIndexMap[raftServer.ID] = raftServer.LastIndex } for _, s := range reply.Servers { if s.Leader { - serverHealthDataLeader, ok := serverHealthDataMap[s.ID] + lastIndex, ok := serverIdLastIndexMap[s.ID] if ok { - leaderLastCommitIndex = serverHealthDataLeader.LastIndex + leaderLastCommitIndex = lastIndex } } } @@ -104,9 +99,9 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { state = "leader" } - serverHealthData, ok := serverHealthDataMap[s.ID] + serverLastIndex, ok := serverIdLastIndexMap[s.ID] if ok { - trailsLeaderBy := leaderLastCommitIndex - serverHealthData.LastIndex + trailsLeaderBy := leaderLastCommitIndex - serverLastIndex trailsLeaderByText := fmt.Sprintf("%d commits", trailsLeaderBy) if s.Leader { trailsLeaderByText = "-" @@ -114,7 +109,7 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { trailsLeaderByText = fmt.Sprintf("%d commit", trailsLeaderBy) } result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v\x1f%s", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverHealthData.LastIndex, trailsLeaderByText)) + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverLastIndex, trailsLeaderByText)) } else { result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "-")) From dc253443d4aabfc6af6e2a6e81c9ab94855b11cb Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Thu, 8 Jun 2023 22:45:28 +0530 Subject: [PATCH 15/24] fix change log --- .changelog/17582.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/17582.txt b/.changelog/17582.txt index dce53703370f..bc4dd7e4e573 100644 --- a/.changelog/17582.txt +++ b/.changelog/17582.txt @@ -1,3 +1,3 @@ ```release-note:feature -raft list-peers shows the number of commits each follower is trailing the leader by to aid in troubleshooting. +consul operator raft list-peers shows the number of commits each follower is trailing the leader by to aid in troubleshooting. ``` From 36317087a9bc9a47d614a94b9e8c4adb5cb58aca Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Thu, 8 Jun 2023 23:21:13 +0530 Subject: [PATCH 16/24] fix tests --- agent/consul/operator_raft_endpoint_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/consul/operator_raft_endpoint_test.go b/agent/consul/operator_raft_endpoint_test.go index 7ce5b6e946c8..e9b991c9654e 100644 --- a/agent/consul/operator_raft_endpoint_test.go +++ b/agent/consul/operator_raft_endpoint_test.go @@ -60,6 +60,7 @@ func TestOperator_RaftGetConfiguration(t *testing.T) { Leader: true, Voter: true, ProtocolVersion: "3", + LastIndex: uint64(14), }, }, Index: future.Index(), From 40dc1e1e834994a0eac6de899cdcb05f89ed3093 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Thu, 8 Jun 2023 23:42:22 +0530 Subject: [PATCH 17/24] fix tests --- agent/consul/operator_raft_endpoint_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/agent/consul/operator_raft_endpoint_test.go b/agent/consul/operator_raft_endpoint_test.go index e9b991c9654e..9dfbc9fb6613 100644 --- a/agent/consul/operator_raft_endpoint_test.go +++ b/agent/consul/operator_raft_endpoint_test.go @@ -50,6 +50,13 @@ func TestOperator_RaftGetConfiguration(t *testing.T) { if len(future.Configuration().Servers) != 1 { t.Fatalf("bad: %v", future.Configuration().Servers) } + + serverIDLastIndexMap := make(map[raft.ServerID]uint64) + + for _, serverState := range s1.autopilot.GetState().Servers { + serverIDLastIndexMap[serverState.Server.ID] = serverState.Stats.LastIndex + } + me := future.Configuration().Servers[0] expected := structs.RaftConfigurationResponse{ Servers: []*structs.RaftServer{ @@ -60,7 +67,7 @@ func TestOperator_RaftGetConfiguration(t *testing.T) { Leader: true, Voter: true, ProtocolVersion: "3", - LastIndex: uint64(14), + LastIndex: serverIDLastIndexMap[me.ID], }, }, Index: future.Index(), From e7afa34d307901b92fff330f1003c60372207bc0 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Fri, 9 Jun 2023 07:21:52 +0530 Subject: [PATCH 18/24] fix test operator raft endpoint test --- agent/consul/operator_raft_endpoint_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/agent/consul/operator_raft_endpoint_test.go b/agent/consul/operator_raft_endpoint_test.go index 9dfbc9fb6613..7242c40e6c45 100644 --- a/agent/consul/operator_raft_endpoint_test.go +++ b/agent/consul/operator_raft_endpoint_test.go @@ -121,6 +121,10 @@ func TestOperator_RaftGetConfiguration_ACLDeny(t *testing.T) { if len(future.Configuration().Servers) != 1 { t.Fatalf("bad: %v", future.Configuration().Servers) } + serverIDLastIndexMap := make(map[raft.ServerID]uint64) + for _, serverState := range s1.autopilot.GetState().Servers { + serverIDLastIndexMap[serverState.Server.ID] = serverState.Stats.LastIndex + } me := future.Configuration().Servers[0] expected := structs.RaftConfigurationResponse{ Servers: []*structs.RaftServer{ @@ -131,6 +135,7 @@ func TestOperator_RaftGetConfiguration_ACLDeny(t *testing.T) { Leader: true, Voter: true, ProtocolVersion: "3", + LastIndex: serverIDLastIndexMap[me.ID], }, }, Index: future.Index(), From 8c1bab490bf5565ed5c0b72d348978ae3e54b69b Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut <134911583+absolutelightning@users.noreply.github.com> Date: Fri, 9 Jun 2023 21:46:19 +0530 Subject: [PATCH 19/24] Update .changelog/17582.txt Co-authored-by: Semir Patel --- .changelog/17582.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/17582.txt b/.changelog/17582.txt index bc4dd7e4e573..122b9df98116 100644 --- a/.changelog/17582.txt +++ b/.changelog/17582.txt @@ -1,3 +1,3 @@ ```release-note:feature -consul operator raft list-peers shows the number of commits each follower is trailing the leader by to aid in troubleshooting. +cli: `consul operator raft list-peers` command shows the number of commits each follower is trailing the leader by to aid in troubleshooting. ``` From 6b76d591b4a21bcd8551360a14be0bd4d35e0131 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Fri, 9 Jun 2023 21:49:22 +0530 Subject: [PATCH 20/24] nits --- .../operator/raft/listpeers/operator_raft_list.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/command/operator/raft/listpeers/operator_raft_list.go b/command/operator/raft/listpeers/operator_raft_list.go index 495b2f64d185..29643a87cf33 100644 --- a/command/operator/raft/listpeers/operator_raft_list.go +++ b/command/operator/raft/listpeers/operator_raft_list.go @@ -99,21 +99,19 @@ func raftListPeers(client *api.Client, stale bool) (string, error) { state = "leader" } + trailsLeaderByText := "-" serverLastIndex, ok := serverIdLastIndexMap[s.ID] if ok { trailsLeaderBy := leaderLastCommitIndex - serverLastIndex - trailsLeaderByText := fmt.Sprintf("%d commits", trailsLeaderBy) + trailsLeaderByText = fmt.Sprintf("%d commits", trailsLeaderBy) if s.Leader { trailsLeaderByText = "-" - } else if trailsLeaderBy <= 1 { + } else if trailsLeaderBy == 1 { trailsLeaderByText = fmt.Sprintf("%d commit", trailsLeaderBy) } - result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v\x1f%s", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverLastIndex, trailsLeaderByText)) - } else { - result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v", - s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, "-")) } + result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v\x1f%s", + s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverLastIndex, trailsLeaderByText)) } return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil From 7f8d9d8a90452e3b2b73e1f20cfa7a0cc727e0bd Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Wed, 14 Jun 2023 20:28:29 +0530 Subject: [PATCH 21/24] updated docs --- website/content/commands/operator/raft.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index e5bf6d1e0c81..25dea30ba62f 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -46,10 +46,10 @@ Usage: `consul operator raft list-peers -stale=[true|false]` The output looks like this: ```text -Node ID Address State Voter RaftProtocol -alice 127.0.0.1:8300 127.0.0.1:8300 follower true 2 -bob 127.0.0.2:8300 127.0.0.2:8300 leader true 3 -carol 127.0.0.3:8300 127.0.0.3:8300 follower true 2 +Node ID Address State Voter RaftProtocol Commit Index Trails Leader By +alice 127.0.0.1:8300 127.0.0.1:8300 follower true 2 1167 0 commits +bob 127.0.0.2:8300 127.0.0.2:8300 leader true 3 1167 - +carol 127.0.0.3:8300 127.0.0.3:8300 follower true 2 1159 8 commits ``` `Node` is the node name of the server, as known to Consul, or "(unknown)" if From ee1deb0c672f650ae03380d1dc239bad1a480e86 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Thu, 15 Jun 2023 08:43:11 +0530 Subject: [PATCH 22/24] explanation added --- website/content/commands/operator/raft.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 25dea30ba62f..24b152ba06c9 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -66,6 +66,10 @@ Raft configuration. `Voter` is "true" or "false", indicating if the server has a vote in the Raft configuration. +`Commit Index` is the last log index the server has a record of in its Raft log. + +`Trails Leader By` is the difference in commits the server from the leader. + #### Command Options - `-stale` - Enables non-leader servers to provide cluster state information. From ba8e3f4195f2659954a3e0443288a9ac332ace09 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Thu, 15 Jun 2023 08:47:09 +0530 Subject: [PATCH 23/24] fix doc --- website/content/commands/operator/raft.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 24b152ba06c9..3c2d8a1221af 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -68,7 +68,7 @@ configuration. `Commit Index` is the last log index the server has a record of in its Raft log. -`Trails Leader By` is the difference in commits the server from the leader. +`Trails Leader By` is the difference in commit index the server has from the leader. #### Command Options From 79830d23c7e38e90d544fc971caf40ea70c39837 Mon Sep 17 00:00:00 2001 From: Ashesh Vidyut Date: Thu, 15 Jun 2023 21:26:59 +0530 Subject: [PATCH 24/24] fix docs --- website/content/commands/operator/raft.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/commands/operator/raft.mdx b/website/content/commands/operator/raft.mdx index 3c2d8a1221af..90f891b6e704 100644 --- a/website/content/commands/operator/raft.mdx +++ b/website/content/commands/operator/raft.mdx @@ -68,7 +68,7 @@ configuration. `Commit Index` is the last log index the server has a record of in its Raft log. -`Trails Leader By` is the difference in commit index the server has from the leader. +`Trails Leader By` is the number of commits a follower trails the leader by. #### Command Options