From 8c32cd96fb2e9c6d9e9184b9fe015921e18f4a09 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Tue, 1 Aug 2017 16:35:15 -0700 Subject: [PATCH 1/5] clientv3: add 'HashKV' to 'Maintenance' interface Signed-off-by: Gyu-Ho Lee --- clientv3/maintenance.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/clientv3/maintenance.go b/clientv3/maintenance.go index 236ab261dd9..33e3553df36 100644 --- a/clientv3/maintenance.go +++ b/clientv3/maintenance.go @@ -28,6 +28,7 @@ type ( AlarmResponse pb.AlarmResponse AlarmMember pb.AlarmMember StatusResponse pb.StatusResponse + HashKVResponse pb.HashKVResponse MoveLeaderResponse pb.MoveLeaderResponse ) @@ -50,6 +51,11 @@ type Maintenance interface { // Status gets the status of the endpoint. Status(ctx context.Context, endpoint string) (*StatusResponse, error) + // HashKV returns a hash of the KV state at the time of the RPC. + // If revision is zero, the hash is computed on all keys. If the revision + // is non-zero, the hash is computed on all keys at or below the given revision. + HashKV(ctx context.Context, endpoint string, rev int64) (*HashKVResponse, error) + // Snapshot provides a reader for a snapshot of a backend. Snapshot(ctx context.Context) (io.ReadCloser, error) @@ -159,6 +165,19 @@ func (m *maintenance) Status(ctx context.Context, endpoint string) (*StatusRespo return (*StatusResponse)(resp), nil } +func (m *maintenance) HashKV(ctx context.Context, endpoint string, rev int64) (*HashKVResponse, error) { + remote, cancel, err := m.dial(endpoint) + if err != nil { + return nil, toErr(ctx, err) + } + defer cancel() + resp, err := remote.HashKV(ctx, &pb.HashKVRequest{Revision: rev}, grpc.FailFast(false)) + if err != nil { + return nil, toErr(ctx, err) + } + return (*HashKVResponse)(resp), nil +} + func (m *maintenance) Snapshot(ctx context.Context) (io.ReadCloser, error) { ss, err := m.remote.Snapshot(ctx, &pb.SnapshotRequest{}, grpc.FailFast(false)) if err != nil { From 9982cd0528c8dd4eb94841e6ae9d7b065399a312 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Tue, 1 Aug 2017 16:45:07 -0700 Subject: [PATCH 2/5] clientv3/integration: add 'TestMaintenanceHashKV' Signed-off-by: Gyu-Ho Lee --- clientv3/integration/maintenance_test.go | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/clientv3/integration/maintenance_test.go b/clientv3/integration/maintenance_test.go index e5496406ed4..27b3b0eab73 100644 --- a/clientv3/integration/maintenance_test.go +++ b/clientv3/integration/maintenance_test.go @@ -23,6 +23,39 @@ import ( "github.com/coreos/etcd/pkg/testutil" ) +func TestMaintenanceHashKV(t *testing.T) { + defer testutil.AfterTest(t) + + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + for i := 0; i < 3; i++ { + if _, err := clus.RandClient().Put(context.Background(), "foo", "bar"); err != nil { + t.Fatal(err) + } + } + + var hv uint32 + for i := 0; i < 3; i++ { + cli := clus.Client(i) + // ensure writes are replicated + if _, err := cli.Get(context.TODO(), "foo"); err != nil { + t.Fatal(err) + } + hresp, err := cli.HashKV(context.Background(), clus.Members[i].GRPCAddr(), 0) + if err != nil { + t.Fatal(err) + } + if hv == 0 { + hv = hresp.Hash + continue + } + if hv != hresp.Hash { + t.Fatalf("#%d: hash expected %d, got %d", i, hv, hresp.Hash) + } + } +} + func TestMaintenanceMoveLeader(t *testing.T) { defer testutil.AfterTest(t) From 5176b63fa0f4dc3f0fec0c010d7d8977fedec2a5 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Tue, 1 Aug 2017 17:13:47 -0700 Subject: [PATCH 3/5] ctlv3: add 'endpoint hashkv' command Signed-off-by: Gyu-Ho Lee --- etcdctl/README.md | 43 +++++++++++++++++++++++++ etcdctl/ctlv3/command/ep_command.go | 41 +++++++++++++++++++++++ etcdctl/ctlv3/command/printer.go | 13 ++++++++ etcdctl/ctlv3/command/printer_fields.go | 9 ++++++ etcdctl/ctlv3/command/printer_json.go | 1 + etcdctl/ctlv3/command/printer_simple.go | 7 ++++ etcdctl/ctlv3/command/printer_table.go | 10 ++++++ 7 files changed, 124 insertions(+) diff --git a/etcdctl/README.md b/etcdctl/README.md index 942d235569f..204536a3351 100644 --- a/etcdctl/README.md +++ b/etcdctl/README.md @@ -641,6 +641,49 @@ Get the status for all endpoints in the cluster associated with the default endp +------------------------+------------------+----------------+---------+-----------+-----------+------------+ ``` +### ENDPOINT HASHKV + +ENDPOINT HASHKV fetches the hash of the key-value store of an endpoint. + +#### Output + +##### Simple format + +Prints a humanized table of each endpoint URL and KV history hash. + +##### JSON format + +Prints a line of JSON encoding each endpoint URL and KV history hash. + +#### Examples + +Get the hash for the default endpoint: + +```bash +./etcdctl endpoint hashkv +# 127.0.0.1:2379, 1084519789 +``` + +Get the status for the default endpoint as JSON: + +```bash +./etcdctl -w json endpoint hashkv +# [{"Endpoint":"127.0.0.1:2379","Hash":{"header":{"cluster_id":14841639068965178418,"member_id":10276657743932975437,"revision":1,"raft_term":3},"hash":1084519789,"compact_revision":-1}}] +``` + +Get the status for all endpoints in the cluster associated with the default endpoint: + +```bash +./etcdctl -w table endpoint --cluster hashkv ++------------------------+------------+ +| ENDPOINT | HASH | ++------------------------+------------+ +| http://127.0.0.1:12379 | 1084519789 | +| http://127.0.0.1:22379 | 1084519789 | +| http://127.0.0.1:32379 | 1084519789 | ++------------------------+------------+ +``` + ### ALARM \ Provides alarm related commands diff --git a/etcdctl/ctlv3/command/ep_command.go b/etcdctl/ctlv3/command/ep_command.go index d31de3c4453..4fac7e44d95 100644 --- a/etcdctl/ctlv3/command/ep_command.go +++ b/etcdctl/ctlv3/command/ep_command.go @@ -28,6 +28,7 @@ import ( ) var epClusterEndpoints bool +var epHashKVRev int64 // NewEndpointCommand returns the cobra command for "endpoint". func NewEndpointCommand() *cobra.Command { @@ -39,6 +40,7 @@ func NewEndpointCommand() *cobra.Command { ec.PersistentFlags().BoolVar(&epClusterEndpoints, "cluster", false, "use all endpoints from the cluster member list") ec.AddCommand(newEpHealthCommand()) ec.AddCommand(newEpStatusCommand()) + ec.AddCommand(newEpHashKVCommand()) return ec } @@ -64,6 +66,16 @@ The items in the lists are endpoint, ID, version, db size, is leader, raft term, } } +func newEpHashKVCommand() *cobra.Command { + hc := &cobra.Command{ + Use: "hashkv", + Short: "Prints the KV history hash for each endpoint in --endpoints", + Run: epHashKVCommandFunc, + } + hc.PersistentFlags().Int64Var(&epHashKVRev, "rev", 0, "maximum revision to hash (default: all revisions)") + return hc +} + // epHealthCommandFunc executes the "endpoint-health" command. func epHealthCommandFunc(cmd *cobra.Command, args []string) { flags.SetPflagsFromEnv("ETCDCTL", cmd.InheritedFlags()) @@ -151,6 +163,35 @@ func epStatusCommandFunc(cmd *cobra.Command, args []string) { } } +type epHashKV struct { + Ep string `json:"Endpoint"` + Resp *v3.HashKVResponse `json:"HashKV"` +} + +func epHashKVCommandFunc(cmd *cobra.Command, args []string) { + c := mustClientFromCmd(cmd) + + hashList := []epHashKV{} + var err error + for _, ep := range endpointsFromCluster(cmd) { + ctx, cancel := commandCtx(cmd) + resp, serr := c.HashKV(ctx, ep, epHashKVRev) + cancel() + if serr != nil { + err = serr + fmt.Fprintf(os.Stderr, "Failed to get the hash of endpoint %s (%v)\n", ep, serr) + continue + } + hashList = append(hashList, epHashKV{Ep: ep, Resp: resp}) + } + + display.EndpointHashKV(hashList) + + if err != nil { + ExitWithError(ExitError, err) + } +} + func endpointsFromCluster(cmd *cobra.Command) []string { if !epClusterEndpoints { endpoints, err := cmd.Flags().GetStringSlice("endpoints") diff --git a/etcdctl/ctlv3/command/printer.go b/etcdctl/ctlv3/command/printer.go index b84dcda74a2..613c555ebe6 100644 --- a/etcdctl/ctlv3/command/printer.go +++ b/etcdctl/ctlv3/command/printer.go @@ -43,6 +43,7 @@ type printer interface { MemberList(v3.MemberListResponse) EndpointStatus([]epStatus) + EndpointHashKV([]epHashKV) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) Alarm(v3.AlarmResponse) @@ -146,6 +147,7 @@ func newPrinterUnsupported(n string) printer { } func (p *printerUnsupported) EndpointStatus([]epStatus) { p.p(nil) } +func (p *printerUnsupported) EndpointHashKV([]epHashKV) { p.p(nil) } func (p *printerUnsupported) DBStatus(dbstatus) { p.p(nil) } func (p *printerUnsupported) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) { p.p(nil) } @@ -184,6 +186,17 @@ func makeEndpointStatusTable(statusList []epStatus) (hdr []string, rows [][]stri return } +func makeEndpointHashKVTable(hashList []epHashKV) (hdr []string, rows [][]string) { + hdr = []string{"endpoint", "hash"} + for _, h := range hashList { + rows = append(rows, []string{ + h.Ep, + fmt.Sprint(h.Resp.Hash), + }) + } + return +} + func makeDBStatusTable(ds dbstatus) (hdr []string, rows [][]string) { hdr = []string{"hash", "revision", "total keys", "total size"} rows = append(rows, []string{ diff --git a/etcdctl/ctlv3/command/printer_fields.go b/etcdctl/ctlv3/command/printer_fields.go index 3e25a9504b7..f7d1cae5915 100644 --- a/etcdctl/ctlv3/command/printer_fields.go +++ b/etcdctl/ctlv3/command/printer_fields.go @@ -146,6 +146,15 @@ func (p *fieldsPrinter) EndpointStatus(eps []epStatus) { } } +func (p *fieldsPrinter) EndpointHashKV(hs []epHashKV) { + for _, h := range hs { + p.hdr(h.Resp.Header) + fmt.Printf("\"Endpoint\" : %q\n", h.Ep) + fmt.Println(`"Hash" :`, h.Resp.Hash) + fmt.Println() + } +} + func (p *fieldsPrinter) Alarm(r v3.AlarmResponse) { p.hdr(r.Header) for _, a := range r.Alarms { diff --git a/etcdctl/ctlv3/command/printer_json.go b/etcdctl/ctlv3/command/printer_json.go index d5d884e5921..19b3a5e688b 100644 --- a/etcdctl/ctlv3/command/printer_json.go +++ b/etcdctl/ctlv3/command/printer_json.go @@ -29,6 +29,7 @@ func newJSONPrinter() printer { } func (p *jsonPrinter) EndpointStatus(r []epStatus) { printJSON(r) } +func (p *jsonPrinter) EndpointHashKV(r []epHashKV) { printJSON(r) } func (p *jsonPrinter) DBStatus(r dbstatus) { printJSON(r) } func printJSON(v interface{}) { diff --git a/etcdctl/ctlv3/command/printer_simple.go b/etcdctl/ctlv3/command/printer_simple.go index 00dda47f773..5e0ae9d3fd2 100644 --- a/etcdctl/ctlv3/command/printer_simple.go +++ b/etcdctl/ctlv3/command/printer_simple.go @@ -136,6 +136,13 @@ func (s *simplePrinter) EndpointStatus(statusList []epStatus) { } } +func (s *simplePrinter) EndpointHashKV(hashList []epHashKV) { + _, rows := makeEndpointHashKVTable(hashList) + for _, row := range rows { + fmt.Println(strings.Join(row, ", ")) + } +} + func (s *simplePrinter) DBStatus(ds dbstatus) { _, rows := makeDBStatusTable(ds) for _, row := range rows { diff --git a/etcdctl/ctlv3/command/printer_table.go b/etcdctl/ctlv3/command/printer_table.go index fb85c5846d3..1aea61a8456 100644 --- a/etcdctl/ctlv3/command/printer_table.go +++ b/etcdctl/ctlv3/command/printer_table.go @@ -44,6 +44,16 @@ func (tp *tablePrinter) EndpointStatus(r []epStatus) { table.SetAlignment(tablewriter.ALIGN_RIGHT) table.Render() } +func (tp *tablePrinter) EndpointHashKV(r []epHashKV) { + hdr, rows := makeEndpointHashKVTable(r) + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader(hdr) + for _, row := range rows { + table.Append(row) + } + table.SetAlignment(tablewriter.ALIGN_RIGHT) + table.Render() +} func (tp *tablePrinter) DBStatus(r dbstatus) { hdr, rows := makeDBStatusTable(r) table := tablewriter.NewWriter(os.Stdout) From 43ccc549fb517e2818f37d209a40464038dee2c4 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Tue, 1 Aug 2017 17:40:26 -0700 Subject: [PATCH 4/5] e2e: test 'endpoint hashkv' command Signed-off-by: Gyu-Ho Lee --- e2e/ctl_v3_endpoint_test.go | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/e2e/ctl_v3_endpoint_test.go b/e2e/ctl_v3_endpoint_test.go index 74a2ebb7a1d..821e77bb7bc 100644 --- a/e2e/ctl_v3_endpoint_test.go +++ b/e2e/ctl_v3_endpoint_test.go @@ -15,12 +15,18 @@ package e2e import ( + "context" + "fmt" "net/url" "testing" + "time" + + "github.com/coreos/etcd/clientv3" ) func TestCtlV3EndpointHealth(t *testing.T) { testCtl(t, endpointHealthTest, withQuorum()) } func TestCtlV3EndpointStatus(t *testing.T) { testCtl(t, endpointStatusTest, withQuorum()) } +func TestCtlV3EndpointHashKV(t *testing.T) { testCtl(t, endpointHashKVTest, withQuorum()) } func endpointHealthTest(cx ctlCtx) { if err := ctlV3EndpointHealth(cx); err != nil { @@ -52,3 +58,35 @@ func ctlV3EndpointStatus(cx ctlCtx) error { } return spawnWithExpects(cmdArgs, eps...) } + +func endpointHashKVTest(cx ctlCtx) { + if err := ctlV3EndpointHashKV(cx); err != nil { + cx.t.Fatalf("endpointHashKVTest ctlV3EndpointHashKV error (%v)", err) + } +} + +func ctlV3EndpointHashKV(cx ctlCtx) error { + eps := cx.epc.EndpointsV3() + + // get latest hash to compare + cli, err := clientv3.New(clientv3.Config{ + Endpoints: eps, + DialTimeout: 3 * time.Second, + }) + if err != nil { + cx.t.Fatal(err) + } + defer cli.Close() + hresp, err := cli.HashKV(context.TODO(), eps[0], 0) + if err != nil { + cx.t.Fatal(err) + } + + cmdArgs := append(cx.PrefixArgs(), "endpoint", "hashkv") + var ss []string + for _, ep := range cx.epc.EndpointsV3() { + u, _ := url.Parse(ep) + ss = append(ss, fmt.Sprintf("%s, %d", u.Host, hresp.Hash)) + } + return spawnWithExpects(cmdArgs, ss...) +} From e4e61479f258fedb2d56a1d613a1c25beae5827a Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Wed, 2 Aug 2017 09:17:57 -0700 Subject: [PATCH 5/5] op-guide/v2-migration: endpoint hashkv post migration Signed-off-by: Gyu-Ho Lee --- Documentation/op-guide/v2-migration.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/op-guide/v2-migration.md b/Documentation/op-guide/v2-migration.md index c03a41a3f68..632a7989326 100644 --- a/Documentation/op-guide/v2-migration.md +++ b/Documentation/op-guide/v2-migration.md @@ -6,7 +6,7 @@ Migrating an application from the API v2 to the API v3 involves two steps: 1) mi ## Migrate client library -API v3 is different from API v2, thus application developers need to use a new client library to send requests to etcd API v3. The documentation of the client v3 is available at https://godoc.org/github.com/coreos/etcd/clientv3. +API v3 is different from API v2, thus application developers need to use a new client library to send requests to etcd API v3. The documentation of the client v3 is available at https://godoc.org/github.com/coreos/etcd/clientv3. There are some notable differences between API v2 and API v3: @@ -38,13 +38,17 @@ Second, migrate the v2 keys into v3 with the [migrate][migrate_command] (`ETCDCT Restart the etcd members and everything should just work. +For etcd v3.3+, run `ETCDCTL_API=3 etcdctl endpoint hashkv --cluster` to ensure key-value stores are consistent post migration. + +**Warn**: When v2 store has expiring TTL keys and migrate command intends to preserve TTLs, migration may be inconsistent with the last committed v2 state when run on any member with a raft index less than the last leader's raft index. + ### Online migration If the application cannot tolerate any downtime, then it must migrate online. The implementation of online migration will vary from application to application but the overall idea is the same. First, write application code using the v3 API. The application must support two modes: a migration mode and a normal mode. The application starts in migration mode. When running in migration mode, the application reads keys using the v3 API first, and, if it cannot find the key, it retries with the API v2. In normal mode, the application only reads keys using the v3 API. The application writes keys over the API v3 in both modes. To acknowledge a switch from migration mode to normal mode, the application watches on a switch mode key. When switch key’s value turns to `true`, the application switches over from migration mode to normal mode. -Second, start a background job to migrate data from the store v2 to the mvcc store by reading keys from the API v2 and writing keys to the API v3. +Second, start a background job to migrate data from the store v2 to the mvcc store by reading keys from the API v2 and writing keys to the API v3. After finishing data migration, the background job writes `true` into the switch mode key to notify the application that it may switch modes.