From 5bba05703c0829cad58ea415a741a1f8514e8599 Mon Sep 17 00:00:00 2001
From: Gyu-Ho Lee <gyuhox@gmail.com>
Date: Wed, 14 Jun 2017 16:08:16 -0700
Subject: [PATCH 1/2] lease: randomize expiry on initial refresh call

Randomize the very first expiry on lease recovery
to prevent recovered leases from expiring all at
the same time.

Address https://github.com/coreos/etcd/issues/8096.

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
---
 lease/lessor.go      | 15 ++++++++++++++-
 lease/lessor_test.go | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/lease/lessor.go b/lease/lessor.go
index 6584a6fb488..df8596ee3c2 100644
--- a/lease/lessor.go
+++ b/lease/lessor.go
@@ -18,6 +18,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"math"
+	"math/rand"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -326,10 +327,22 @@ func (le *lessor) Promote(extend time.Duration) {
 
 	// refresh the expiries of all leases.
 	for _, l := range le.leaseMap {
-		l.refresh(extend)
+		// randomize expiry with 士10%, otherwise leases of same TTL
+		// will expire all at the same time,
+		l.refresh(extend + computeRandomDelta(l.ttl))
 	}
 }
 
+func computeRandomDelta(seconds int64) time.Duration {
+	var delta int64
+	if seconds > 10 {
+		delta = int64(float64(seconds) * 0.1 * rand.Float64())
+	} else {
+		delta = rand.Int63n(10)
+	}
+	return time.Duration(delta) * time.Second
+}
+
 func (le *lessor) Demote() {
 	le.mu.Lock()
 	defer le.mu.Unlock()
diff --git a/lease/lessor_test.go b/lease/lessor_test.go
index bfada89932e..93ea91c881e 100644
--- a/lease/lessor_test.go
+++ b/lease/lessor_test.go
@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/coreos/etcd/mvcc/backend"
+	"github.com/coreos/etcd/pkg/monotime"
 )
 
 const (
@@ -210,6 +211,41 @@ func TestLessorRenew(t *testing.T) {
 	}
 }
 
+// TestLessorRenewRandomize ensures Lessor renews with randomized expiry.
+func TestLessorRenewRandomize(t *testing.T) {
+	dir, be := NewTestBackend(t)
+	defer os.RemoveAll(dir)
+
+	le := newLessor(be, minLeaseTTL)
+	for i := LeaseID(1); i <= 10; i++ {
+		if _, err := le.Grant(i, 3600); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// simulate stop and recovery
+	le.Stop()
+	be.Close()
+	bcfg := backend.DefaultBackendConfig()
+	bcfg.Path = filepath.Join(dir, "be")
+	be = backend.New(bcfg)
+	defer be.Close()
+	le = newLessor(be, minLeaseTTL)
+
+	now := monotime.Now()
+
+	// extend after recovery should randomize expiries
+	le.Promote(0)
+
+	for _, l := range le.leaseMap {
+		leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9))
+		pc := (float64(leftSeconds-3600) / float64(3600)) * 100
+		if pc > 10.0 || pc < -10.0 || pc == 0 { // should be within 士10%
+			t.Fatalf("expected randomized expiry, got %d seconds (ttl: 3600)", leftSeconds)
+		}
+	}
+}
+
 func TestLessorDetach(t *testing.T) {
 	dir, be := NewTestBackend(t)
 	defer os.RemoveAll(dir)

From 95bc33f37f7c31a4cd06287d44879a60baaee40c Mon Sep 17 00:00:00 2001
From: Gyu-Ho Lee <gyuhox@gmail.com>
Date: Thu, 15 Jun 2017 01:52:05 -0700
Subject: [PATCH 2/2] integration: remove lease exist checking on randomized
 expiry

Lease with TTL 5 should be renewed with randomization,
thus it's still possible to exist after 3 seconds.

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
---
 integration/v3_lease_test.go | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/integration/v3_lease_test.go b/integration/v3_lease_test.go
index dbb8e6b727a..3846817ea7f 100644
--- a/integration/v3_lease_test.go
+++ b/integration/v3_lease_test.go
@@ -66,16 +66,10 @@ func TestV3LeasePrmote(t *testing.T) {
 	// it was going to expire anyway.
 	time.Sleep(3 * time.Second)
 
+	// expiring lease should be renewed with randomized delta
 	if !leaseExist(t, clus, lresp.ID) {
 		t.Error("unexpected lease not exists")
 	}
-
-	// let lease expires. total lease = 5 seconds and we already
-	// waits for 3 seconds, so 3 seconds more is enough.
-	time.Sleep(3 * time.Second)
-	if leaseExist(t, clus, lresp.ID) {
-		t.Error("unexpected lease exists")
-	}
 }
 
 // TestV3LeaseRevoke ensures a key is deleted once its lease is revoked.