Skip to content

Commit

Permalink
Merge pull request #250 from openshift-cherrypick-robot/cherry-pick-2…
Browse files Browse the repository at this point in the history
…49-to-release-0.6

[release-0.6] Fix lease handling for healthy nodes in case not owned lease exists
  • Loading branch information
slintes authored Aug 31, 2023
2 parents 723a4ca + 12e24c5 commit 359fc6c
Show file tree
Hide file tree
Showing 6 changed files with 60 additions and 6 deletions.
47 changes: 47 additions & 0 deletions controllers/nodehealthcheck_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -620,6 +620,53 @@ var _ = Describe("Node Health Check CR", func() {
}, "2s", "100ms").Should(BeTrue(), "lease wasn't removed")

})

It("node lease not owned by us isn't removed, but status is updated (invalidate lease error is ignored)", func() {
cr := newRemediationCR(unhealthyNodeName, underTest)
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(cr), cr)
Expect(err).ToNot(HaveOccurred())
//Verify lease exist
lease := &coordv1.Lease{}
err = k8sClient.Get(context.Background(), client.ObjectKey{Name: leaseName, Namespace: leaseNs}, lease)
Expect(err).ToNot(HaveOccurred())

// change lease owner
newLeaseOwner := "someone-else"
lease.Spec.HolderIdentity = pointer.String(newLeaseOwner)
Expect(k8sClient.Update(context.Background(), lease)).To(Succeed(), "failed to update lease owner")

//Mock node becoming healthy
node := &v1.Node{}
err = k8sClient.Get(context.Background(), client.ObjectKey{Name: unhealthyNodeName}, node)
Expect(err).ToNot(HaveOccurred())
for i, c := range node.Status.Conditions {
if c.Type == v1.NodeReady {
node.Status.Conditions[i].Status = v1.ConditionTrue
}
}
err = k8sClient.Status().Update(context.Background(), node)
Expect(err).ToNot(HaveOccurred())

//Remediation should be removed
Eventually(func() bool {
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(cr), cr)
return errors.IsNotFound(err)
}, "2s", "100ms").Should(BeTrue(), "remediation CR wasn't removed")

// Status should be updated even though lease isn't owned by us anymore
Eventually(func(g Gomega) {
g.Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(underTest), underTest)).To(Succeed())
g.Expect(underTest.Status.InFlightRemediations).To(BeEmpty())
g.Expect(underTest.Status.UnhealthyNodes).To(BeEmpty())
}, "2s", "100ms").Should(Succeed(), "status update failed")

//Verify NHC didn't touch the lease
Consistently(func(g Gomega) {
g.Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: leaseName, Namespace: leaseNs}, lease)).To(Succeed(), "failed to get lease")
g.Expect(*lease.Spec.HolderIdentity).To(Equal(newLeaseOwner))
}, "2s", "100ms").Should(Succeed(), "lease was touched even though it's not owned by us")

})
})

When("an unhealthy node lease is already taken", func() {
Expand Down
4 changes: 4 additions & 0 deletions controllers/resources/lease.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@ func (m *nhcLeaseManager) InvalidateLease(ctx context.Context, nodeName string)
}
err := m.commonLeaseManager.InvalidateLease(ctx, node)
if err != nil {
if _, ok := err.(lease.AlreadyHeldError); ok {
// lease exists but isn't owned by us, can be ignored
return nil
}
m.log.Error(err, "failed to invalidate lease", "node name", nodeName)
return err
}
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.20

require (
github.com/go-logr/logr v1.2.4
github.com/medik8s/common v1.7.0
github.com/medik8s/common v1.8.0
github.com/onsi/ginkgo/v2 v2.9.5
github.com/onsi/gomega v1.27.7
github.com/openshift/api v0.0.0-20221013123534-96eec44e1979 // release-4.11
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -235,8 +235,8 @@ github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/medik8s/common v1.7.0 h1:JwimhWigPTAszGG2jYJmh+uVHq2nFUPRRljw7ZhlQhg=
github.com/medik8s/common v1.7.0/go.mod h1:ZT/3hfMXJLmZEcqmxRWB5LGC8Wl+qKGGQ4zM8hOE7PY=
github.com/medik8s/common v1.8.0 h1:7TOH+XQEvBEMpTZEewVtUzCpjGosaklWPt8E4gC2PNk=
github.com/medik8s/common v1.8.0/go.mod h1:ZT/3hfMXJLmZEcqmxRWB5LGC8Wl+qKGGQ4zM8hOE7PY=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
Expand Down
7 changes: 5 additions & 2 deletions vendor/github.com/medik8s/common/pkg/lease/manager.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion vendor/modules.txt
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ github.com/mailru/easyjson/jwriter
# github.com/matttproud/golang_protobuf_extensions v1.0.4
## explicit; go 1.9
github.com/matttproud/golang_protobuf_extensions/pbutil
# github.com/medik8s/common v1.7.0
# github.com/medik8s/common v1.8.0
## explicit; go 1.20
github.com/medik8s/common/pkg/annotations
github.com/medik8s/common/pkg/conditions
Expand Down

0 comments on commit 359fc6c

Please sign in to comment.