Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Dynamiclly set safe time to assume node rebooted seconds #197

Closed
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 41 additions & 3 deletions pkg/reboot/calculator.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@ package reboot

import (
"context"
"errors"
"fmt"
"strconv"
"time"

"github.com/go-logr/logr"
"github.com/medik8s/common/pkg/events"
commonlabels "github.com/medik8s/common/pkg/labels"
pkgerrors "github.com/pkg/errors"

v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
Expand Down Expand Up @@ -76,12 +78,14 @@ func NewManagerSafeTimeCalculator(k8sClient client.Client) SafeTimeCalculator {
}

func (s *safeTimeCalculator) GetTimeToAssumeNodeRebooted() time.Duration {
minTime := s.minTimeToAssumeNodeRebooted
mshitrit marked this conversation as resolved.
Show resolved Hide resolved
if !s.isAgent {
return s.timeToAssumeNodeRebooted
//TODO mshitrit handle error
minTime, _ = s.getMinSafeTimeFromConfig()
}

if s.timeToAssumeNodeRebooted < s.minTimeToAssumeNodeRebooted {
return s.minTimeToAssumeNodeRebooted
if s.timeToAssumeNodeRebooted < minTime {
return minTime
}
return s.timeToAssumeNodeRebooted
}
Expand Down Expand Up @@ -153,6 +157,7 @@ func (s *safeTimeCalculator) manageSafeRebootTimeInConfiguration(minTime time.Du

if snrConf.Spec.SafeTimeToAssumeNodeRebootedSeconds < minTimeSec {
isUpdateRequired = true
//TODO mshitrit don't update the spec but create a warning in the status instead
snrConf.Spec.SafeTimeToAssumeNodeRebootedSeconds = minTimeSec
s.SetTimeToAssumeNodeRebooted(time.Duration(minTimeSec) * time.Second)
msg, reason := "Automatic update since value isn't valid anymore", "SafeTimeToAssumeNodeRebootedSecondsModified"
Expand All @@ -175,6 +180,39 @@ func (s *safeTimeCalculator) manageSafeRebootTimeInConfiguration(minTime time.Du
return nil
}

func (s *safeTimeCalculator) getMinSafeTimeFromConfig() (time.Duration, error) {
//TODO mshitrit add some logs
confList := &v1alpha1.SelfNodeRemediationConfigList{}
if err := s.k8sClient.List(context.Background(), confList); err != nil {
mshitrit marked this conversation as resolved.
Show resolved Hide resolved
s.log.Error(err, "failed to get snr configuration")

return 0, err
}

var config v1alpha1.SelfNodeRemediationConfig
if len(confList.Items) < 1 {
return 0, errors.New("SNR config not found")
} else {
config = confList.Items[0]
mshitrit marked this conversation as resolved.
Show resolved Hide resolved
}

if config.GetAnnotations() == nil {
return 0, errors.New("failed getting MinSafeTimeFromConfig config does not contain annotations")
}

minTimeSecString, isFound := config.GetAnnotations()[utils.MinSafeTimeAnnotation]
if !isFound {
return 0, fmt.Errorf("failed getting MinSafeTimeFromConfig config does not contain %s annotation", utils.MinSafeTimeAnnotation)
}

minTimeSec, err := strconv.Atoi(minTimeSecString)
if err != nil {
return 0, pkgerrors.Wrapf(err, "failed getting MinSafeTimeFromConfig")
}
return time.Duration(minTimeSec) * time.Second, nil

}

func (s *safeTimeCalculator) IsAgent() bool {
return s.isAgent
}
Expand Down