Skip to content

Commit

Permalink
Addition of Node prep flag to install and the injection of the init c…
Browse files Browse the repository at this point in the history 
…ontainer into the linux daemonset (#1712)

injection of container when nodePrep flag is set to iSCSI

---------

Co-authored-by: Emma Hardison <[email protected]>
  • Loading branch information
@brownaaron @emmahardison
brownaaron and emmahardison committed Sep 11, 2024
1 parent 09b9dd3 commit 676e236
Show file tree
Hide file tree
Showing 13 changed files with 377 additions and 38 deletions.
19 changes: 12 additions & 7 deletions cli/cmd/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/netapp/trident/cli/api"
k8sclient "github.com/netapp/trident/cli/k8s_client"
tridentconfig "github.com/netapp/trident/config"
"github.com/netapp/trident/internal/nodeprep"
. "github.com/netapp/trident/logging"
"github.com/netapp/trident/utils"
"github.com/netapp/trident/utils/crypto"
Expand Down Expand Up @@ -99,7 +100,7 @@ var (
silent bool
useIPv6 bool
silenceAutosupport bool
enableNodePrep bool
nodePrep []string
skipK8sVersionCheck bool
windows bool
enableForceDetach bool
Expand Down Expand Up @@ -200,8 +201,7 @@ func init() {
installCmd.Flags().BoolVar(&useIPv6, "use-ipv6", false, "Use IPv6 for Trident's communication.")
installCmd.Flags().BoolVar(&silenceAutosupport, "silence-autosupport", tridentconfig.BuildType != "stable",
"Don't send autosupport bundles to NetApp automatically.")
installCmd.Flags().BoolVar(&enableNodePrep, "enable-node-prep", false,
"(Deprecated) Attempt to automatically install required packages on nodes.")
installCmd.Flags().StringSliceVar(&nodePrep, "node-prep", []string{}, "Comma separated list of protocols to prepare nodes for. Currently only iSCSI is supported.")
installCmd.Flags().BoolVar(&enableForceDetach, "enable-force-detach", false,
"Enable the force detach feature.")
installCmd.Flags().BoolVar(&disableAuditLog, "disable-audit-log", true, "Disable the audit logger.")
Expand Down Expand Up @@ -265,10 +265,6 @@ func init() {
if err := installCmd.Flags().MarkHidden("autosupport-hostname"); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
}
if err := installCmd.Flags().MarkDeprecated("enable-node-prep",
"enable-node-prep is disabled; flag may be removed in a future release."); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
}
}

var installCmd = &cobra.Command{
Expand Down Expand Up @@ -429,6 +425,7 @@ func processInstallationArguments(_ *cobra.Command) {

persistentObjectLabelKey = TridentPersistentObjectLabelKey
persistentObjectLabelValue = TridentPersistentObjectLabelValue
nodePrep = nodeprep.FormatProtocols(nodePrep)
}

func validateInstallationArguments() error {
Expand All @@ -439,6 +436,10 @@ func validateInstallationArguments() error {
return fmt.Errorf("'%s' is not a valid namespace name; %s", TridentPodNamespace, labelFormat)
}

if err := nodeprep.ValidateProtocols(nodePrep); err != nil {
return err
}

switch logFormat {
case "text", "json":
break
Expand Down Expand Up @@ -703,6 +704,7 @@ func prepareYAMLFiles() error {
ImagePullPolicy: imagePullPolicy,
ISCSISelfHealingInterval: iscsiSelfHealingInterval.String(),
ISCSISelfHealingWaitTime: iscsiSelfHealingWaitTime.String(),
NodePrep: nodePrep,
}
daemonSetYAML := k8sclient.GetCSIDaemonSetYAMLLinux(daemonArgs)
if err = writeFile(daemonsetPath, daemonSetYAML); err != nil {
Expand Down Expand Up @@ -778,6 +780,7 @@ func prepareYAMLFiles() error {
HTTPRequestTimeout: httpRequestTimeout.String(),
ServiceAccountName: getNodeRBACResourceName(true),
ImagePullPolicy: imagePullPolicy,
NodePrep: nodePrep,
}
windowsDaemonSetYAML := k8sclient.GetCSIDaemonSetYAMLWindows(daemonArgs)
if err = writeFile(windowsDaemonSetPath, windowsDaemonSetYAML); err != nil {
Expand Down Expand Up @@ -1169,6 +1172,7 @@ func installTrident() (returnError error) {
HTTPRequestTimeout: httpRequestTimeout.String(),
ServiceAccountName: getNodeRBACResourceName(true),
ImagePullPolicy: imagePullPolicy,
NodePrep: nodePrep,
}
returnError = client.CreateObjectByYAML(
k8sclient.GetCSIDaemonSetYAMLWindows(daemonSetArgs))
Expand Down Expand Up @@ -1223,6 +1227,7 @@ func installTrident() (returnError error) {
ImagePullPolicy: imagePullPolicy,
ISCSISelfHealingInterval: iscsiSelfHealingInterval.String(),
ISCSISelfHealingWaitTime: iscsiSelfHealingWaitTime.String(),
NodePrep: nodePrep,
}
returnError = client.CreateObjectByYAML(
k8sclient.GetCSIDaemonSetYAMLLinux(daemonSetArgs))
Expand Down
59 changes: 32 additions & 27 deletions cli/cmd/install_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,43 +230,48 @@ spec:

func TestValidateInstallationArguments(t *testing.T) {
tests := []struct {
TridentPodNamespace string
name string
tridentPodNamespace string
logFormat string
imagePullPolicy string
cloudProvider string
cloudIdentity string
Valid bool
nodePrep []string
assertValid assert.ErrorAssertionFunc
}{
// Valid arguments
{"default", "text", "IfNotPresent", "", "", true},
{"test-namespace", "text", "IfNotPresent", "", "", true},
{"test-namespace", "json", "Never", "", "", true},
{"test-namespace", "json", "Never", k8sclient.CloudProviderAzure, "", true},
{"test", "text", "Always", k8sclient.CloudProviderAzure, k8sclient.AzureCloudIdentityKey + " a8rry78r8-7733-49bd-6656582", true},
{"test", "text", "Always", k8sclient.CloudProviderAzure, "", true},
{"default namespace", "default", "text", "IfNotPresent", "", "", nil, assert.NoError},
{"test namespace", "test-namespace", "text", "IfNotPresent", "", "", []string{}, assert.NoError},
{"image pull never", "test-namespace", "json", "Never", "", "", []string{"iscsi"}, assert.NoError},
{"cloud provider azure", "test-namespace", "json", "Never", k8sclient.CloudProviderAzure, "", []string{"iscsi"}, assert.NoError},
{"azure cloud identity key", "test", "text", "Always", k8sclient.CloudProviderAzure, k8sclient.AzureCloudIdentityKey + " a8rry78r8-7733-49bd-6656582", []string{}, assert.NoError},
{"image pull always", "test", "text", "Always", k8sclient.CloudProviderAzure, "", []string{}, assert.NoError},

// Invalid arguments
{"", "", "", "", "", false},
{"test", "html", "", "", "", false},
{"test", "text", "Anyways", "", "", false},
{"test", "json", "Never", "Docker", "", false},
{"test", "json", "Never", "Docker", k8sclient.AzureCloudIdentityKey + " a8rry78r8-7733-49bd-6656582", false},
{"test", "text", "IfNotPresent", k8sclient.CloudProviderAWS, "", false},
{"test", "text", "IfNotPresent", "", k8sclient.AzureCloudIdentityKey + " a8rry78r8-7733-49bd-6656582", false},
{"test", "text", "Always", k8sclient.CloudProviderAzure, "a8rry78r8-7733-49bd-6656582", false},
{"invalid namespace", "", "", "", "", "", []string{}, assert.Error},
{"invalid log format", "test", "html", "", "", "", []string{}, assert.Error},
{"invalid image pull policy", "test", "text", "Anyways", "", "", []string{}, assert.Error},
{"invalid cloud provider", "test", "json", "Never", "Docker", "", []string{}, assert.Error},
{"invalid cloud provider with azure key", "test", "json", "Never", "Docker", k8sclient.AzureCloudIdentityKey + " a8rry78r8-7733-49bd-6656582", []string{}, assert.Error},
{"invalid blank cloud identity", "test", "text", "IfNotPresent", k8sclient.CloudProviderAWS, "", []string{}, assert.Error},
{"invalid blank cloud provider", "test", "text", "IfNotPresent", "", k8sclient.AzureCloudIdentityKey + " a8rry78r8-7733-49bd-6656582", []string{}, assert.Error},
{"invalid cloud identity", "test", "text", "Always", k8sclient.CloudProviderAzure, "a8rry78r8-7733-49bd-6656582", []string{}, assert.Error},
{"invalid blank node prep", "default", "text", "IfNotPresent", "", "", []string{""}, assert.Error},
{"invalid only node prep", "default", "text", "IfNotPresent", "", "", []string{"NVME"}, assert.Error},
{"invalid node prep in list", "default", "text", "IfNotPresent", "", "", []string{"iscsi", "nvme"}, assert.Error},
}

for _, test := range tests {
TridentPodNamespace = test.TridentPodNamespace
logFormat = test.logFormat
imagePullPolicy = test.imagePullPolicy
cloudProvider = test.cloudProvider
cloudIdentity = test.cloudIdentity
err := validateInstallationArguments()
if test.Valid {
assert.NoError(t, err, "should be valid")
} else {
assert.Error(t, err, "should be invalid")
}
t.Run(test.name, func(t *testing.T) {
// set global variables that are used in validateInstallationArguments
TridentPodNamespace = test.tridentPodNamespace
logFormat = test.logFormat
imagePullPolicy = test.imagePullPolicy
cloudProvider = test.cloudProvider
cloudIdentity = test.cloudIdentity
nodePrep = test.nodePrep
err := validateInstallationArguments()
test.assertValid(t, err)
})
}
}
1 change: 1 addition & 0 deletions cli/k8s_client/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ type DaemonsetYAMLArguments struct {
ImagePullPolicy string `json:"imagePullPolicy"`
ISCSISelfHealingInterval string `json:"iscsiSelfHealingInterval"`
ISCSISelfHealingWaitTime string `json:"iscsiSelfHealingWaitTime"`
NodePrep []string `json:"nodePrep"`
}

type TridentVersionPodYAMLArguments struct {
Expand Down
59 changes: 59 additions & 0 deletions cli/k8s_client/yaml_factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package k8sclient

import (
"fmt"
"regexp"
"strconv"
"strings"

Expand Down Expand Up @@ -880,6 +881,8 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
}

kubeletDir := strings.TrimRight(args.KubeletDir, "/")
// NodePrep this must come first because it adds a section that has tags in it
daemonSetYAML = replaceNodePrepTag(daemonSetYAML, args.NodePrep)
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage)
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName)
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{CSI_SIDECAR_REGISTRY}", args.ImageRegistry)
Expand Down Expand Up @@ -912,6 +915,61 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
return daemonSetYAML
}

func replaceNodePrepTag(daemonSetYAML string, nodePrep []string) string {
if len(nodePrep) > 0 {
prepareNodeLinuxFormatted := indentSection(daemonSetYAML, prepareNodeLinux, "{PREPARE_NODE}")
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{PREPARE_NODE}", prepareNodeLinuxFormatted)
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{NODE_PREP}", strings.Join(nodePrep, ","))
} else {
daemonSetYAML = removeSectionTag(daemonSetYAML, "{PREPARE_NODE}")
}
return daemonSetYAML
}

func removeSectionTag(daemonSetYAML, tag string) string {
r := regexp.MustCompile("\n +" + tag + "\n")
return string(r.ReplaceAll([]byte(daemonSetYAML), []byte("\n")))
}

func indentSection(daemonSetYAML, section, tag string) (indentedSection string) {
indent := getSectionIndentation(daemonSetYAML, tag)
indentedSection = strings.ReplaceAll(section, "{INDENT}", indent)
return
}

func getSectionIndentation(daemonSetYAML, tag string) (indent string) {
tagIdx := strings.Index(daemonSetYAML, tag)
lineStartIdx := strings.LastIndex(daemonSetYAML[0:tagIdx], "\n") + 1
return daemonSetYAML[lineStartIdx:tagIdx]
}

const prepareNodeLinux = `initContainers:
{INDENT}- name: trident-prepare-node
{INDENT} securityContext:
{INDENT} privileged: true
{INDENT} allowPrivilegeEscalation: true
{INDENT} capabilities:
{INDENT} drop:
{INDENT} - all
{INDENT} add:
{INDENT} - SYS_ADMIN
{INDENT} image: {TRIDENT_IMAGE}
{INDENT} imagePullPolicy: {IMAGE_PULL_POLICY}
{INDENT} command:
{INDENT} - /node_prep
{INDENT} args:
{INDENT} - "--node-prep={NODE_PREP}"
{INDENT} - "--log-format={LOG_FORMAT}"
{INDENT} - "--log-level={LOG_LEVEL}"
{INDENT} {DEBUG}
{INDENT} env:
{INDENT} - name: PATH
{INDENT} value: /netapp:/bin
{INDENT} volumeMounts:
{INDENT} - name: host-dir
{INDENT} mountPath: /host
{INDENT} mountPropagation: "Bidirectional"`

const daemonSet120YAMLTemplateLinux = `---
apiVersion: apps/v1
kind: DaemonSet
Expand All @@ -936,6 +994,7 @@ spec:
hostPID: true
dnsPolicy: ClusterFirstWithHostNet
priorityClassName: system-node-critical
{PREPARE_NODE}
containers:
- name: trident-main
securityContext:
Expand Down
56 changes: 56 additions & 0 deletions cli/k8s_client/yaml_factory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -644,6 +644,62 @@ func TestGetCSIDaemonSetYAMLLinuxImagePullPolicy(t *testing.T) {
}
}

func TestGetCSIDaemonSetYAMLLinux_NodePrep(t *testing.T) {
defaultVersions := []string{"1.26.0"}
expected := "initContainers:"
expectedParam := "--node-prep=%s"

tests := []struct {
Name string
Versions []string
NodePrep []string
ComparisonFunc assert.ComparisonAssertionFunc
ExpectedParam string
}{
{
Name: "nodePrep nil",
Versions: defaultVersions,
NodePrep: nil,
ComparisonFunc: assert.NotContains,
},
{
Name: "nodePrep empty",
Versions: defaultVersions,
NodePrep: []string{},
ComparisonFunc: assert.NotContains,
},
{
Name: "nodePrep list",
Versions: defaultVersions,
NodePrep: []string{"iSCSI", "NVME"},
ComparisonFunc: assert.Contains,
ExpectedParam: fmt.Sprintf(expectedParam, "iSCSI,NVME"),
},
{
Name: "nodePrep single",
Versions: defaultVersions,
NodePrep: []string{"iSCSI"},
ComparisonFunc: assert.Contains,
ExpectedParam: fmt.Sprintf(expectedParam, "iSCSI"),
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
for _, versionString := range test.Versions {
version := versionutils.MustParseSemantic(versionString)
daemonsetArgs := &DaemonsetYAMLArguments{Version: version, NodePrep: test.NodePrep}
yamlData := GetCSIDaemonSetYAMLLinux(daemonsetArgs)
_, err := yaml.YAMLToJSON([]byte(yamlData))
assert.Nilf(t, err, "expected valid YAML for version %s", versionString)
test.ComparisonFunc(t, yamlData, expected)
assert.Contains(t, yamlData, test.ExpectedParam)
// this tag should never be left in the yaml
assert.NotContains(t, yamlData, "{PREPARE_NODE}")
}
})
}
}

func TestGetCSIDaemonSetYAMLLinux_NodeSelectors(t *testing.T) {
daemonsetArgs := &DaemonsetYAMLArguments{
NodeSelector: map[string]string{"node-label-key": "test1"},
Expand Down
71 changes: 70 additions & 1 deletion cmd/node_prep/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,75 @@

package main

import (
"flag"
"fmt"
"os"
"strings"

"github.com/netapp/trident/config"
"github.com/netapp/trident/internal/nodeprep"
. "github.com/netapp/trident/logging"
)

func main() {
println("Executing trident node prep")
flags := initFlags()

initLogging(flags)

Log().WithFields(LogFields{
"version": config.OrchestratorVersion.String(),
"build_time": config.BuildTime,
"binary": os.Args[0],
}).Info("Running Trident node preparation.")

nodeprep.PrepareNode(strings.Split(strings.ToLower(*flags.nodePrep), ","))
}

func initLogging(flags appFlags) {
if err := setLoggingFlags(flags); err != nil {
_, _ = fmt.Fprint(os.Stderr, err)
println("Failed to initialize logging, attempting default settings")
*flags.logLevel = "info"
*flags.logFormat = "text"
if err = setLoggingFlags(flags); err != nil {
_, _ = fmt.Fprint(os.Stderr, err)
println("Failed to initialize logging")
}
}
}

func setLoggingFlags(flags appFlags) error {
// if debug override log level
if *flags.debug {
*flags.logLevel = "debug"
}

if err := InitLogLevel(*flags.logLevel); err != nil {
return err
}

if err := InitLogFormat(*flags.logFormat); err != nil {
return err
}

return nil
}

type appFlags struct {
logFormat *string
logLevel *string
debug *bool
nodePrep *string
}

func initFlags() appFlags {
flags := appFlags{
nodePrep: flag.String("node-prep", "", "List of protocols to prepare node with"),
logFormat: flag.String("log-format", "text", "Logging format (text, json)"),
logLevel: flag.String("log-level", "info", "Logging level (trace, debug, info, warn, error, fatal)"),
debug: flag.Bool("debug", false, "Enable debugging output"),
}
flag.Parse()
return flags
}
Loading

0 comments on commit 676e236

Please sign in to comment.