lease: randomize expiry on initial refresh call

integration/v3_lease_test.go

@@ -66,16 +66,10 @@ func TestV3LeasePrmote(t *testing.T) {
 	// it was going to expire anyway.
 	time.Sleep(3 * time.Second)
 
+	// expiring lease should be renewed with randomized delta
 	if !leaseExist(t, clus, lresp.ID) {
 		t.Error("unexpected lease not exists")
 	}
-
-	// let lease expires. total lease = 5 seconds and we already
-	// waits for 3 seconds, so 3 seconds more is enough.
-	time.Sleep(3 * time.Second)
-	if leaseExist(t, clus, lresp.ID) {
-		t.Error("unexpected lease exists")
-	}
 }
 
 // TestV3LeaseRevoke ensures a key is deleted once its lease is revoked.

lease/lessor.go

@@ -18,6 +18,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"math"
+	"math/rand"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -326,10 +327,22 @@ func (le *lessor) Promote(extend time.Duration) {
 
 	// refresh the expiries of all leases.
 	for _, l := range le.leaseMap {
-		l.refresh(extend)
+		// randomize expiry with 士10%, otherwise leases of same TTL
+		// will expire all at the same time,
+		l.refresh(extend + computeRandomDelta(l.ttl))
 	}
 }
 
+func computeRandomDelta(seconds int64) time.Duration {
+	var delta int64
+	if seconds > 10 {
+		delta = int64(float64(seconds) * 0.1 * rand.Float64())
+	} else {
+		delta = rand.Int63n(10)
+	}
+	return time.Duration(delta) * time.Second
+}
+
 func (le *lessor) Demote() {
 	le.mu.Lock()
 	defer le.mu.Unlock()

lease/lessor_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/coreos/etcd/mvcc/backend"
+	"github.com/coreos/etcd/pkg/monotime"
 )
 
 const (
@@ -210,6 +211,41 @@ func TestLessorRenew(t *testing.T) {
 	}
 }
 
+// TestLessorRenewRandomize ensures Lessor renews with randomized expiry.
+func TestLessorRenewRandomize(t *testing.T) {
+	dir, be := NewTestBackend(t)
+	defer os.RemoveAll(dir)
+
+	le := newLessor(be, minLeaseTTL)
+	for i := LeaseID(1); i <= 10; i++ {
+		if _, err := le.Grant(i, 3600); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// simulate stop and recovery
+	le.Stop()
+	be.Close()
+	bcfg := backend.DefaultBackendConfig()
+	bcfg.Path = filepath.Join(dir, "be")
+	be = backend.New(bcfg)
+	defer be.Close()
+	le = newLessor(be, minLeaseTTL)
+
+	now := monotime.Now()
+
+	// extend after recovery should randomize expiries
+	le.Promote(0)
+
+	for _, l := range le.leaseMap {
+		leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9))
+		pc := (float64(leftSeconds-3600) / float64(3600)) * 100
+		if pc > 10.0 || pc < -10.0 || pc == 0 { // should be within 士10%
+			t.Fatalf("expected randomized expiry, got %d seconds (ttl: 3600)", leftSeconds)
+		}
+	}
+}
+
 func TestLessorDetach(t *testing.T) {
 	dir, be := NewTestBackend(t)
 	defer os.RemoveAll(dir)