Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Get Cluster Information for Running FA on AWS and Test Nodes Status #32

Merged
merged 10 commits into from
May 7, 2023
Merged

Get Cluster Information for Running FA on AWS and Test Nodes Status #32

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c7314ae
Get cluster information for AWS E2E
razo7 May 2, 2023
1d3aed4
Add e2e test - FAR logs
razo7 May 2, 2023
2c6e157
Build shared and node parameters for FAR CR based on cluster type
razo7 May 2, 2023
e7caa97
Run fence_aws with status on far-install namespace
razo7 May 2, 2023
1427bb4
Get the instance ID from the provider ID in AWS
razo7 May 2, 2023
129ac60
Fetch the namespace the operator is installed
razo7 May 2, 2023
f78fccd
Delete redundant function
razo7 May 2, 2023
b50db0d
Better import name and signal missing env var
razo7 May 4, 2023
0497740
Don't expose cluster secrets
razo7 May 4, 2023
0d9572b
Typo fix + use log
razo7 May 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 59 additions & 67 deletions test/e2e/far_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,13 @@ import (
)

const (
fenceAgentDummyName = "echo"
testNamespace = "openshift-operators"
fenceAgentAWS = "fence_aws"
fenceAgentIPMI = "fence_ipmilan"
fenceAgentAction = "status"
nodeIndex = 0
fenceAgentDummyName = "echo"
testNamespace = "far-install"
fenceAgentAWS = "fence_aws"
fenceAgentIPMI = "fence_ipmilan"
fenceAgentAction = "status"
nodeIndex = 0
suceessStatusMessage = "ON"
razo7 marked this conversation as resolved.
Show resolved Hide resolved

// eventually parameters
timeoutLogs = 1 * time.Minute
Expand All @@ -42,13 +43,10 @@ var _ = Describe("FAR E2e", func() {
err error
)
BeforeEach(func() {
// command -> oc get Infrastructure.config.openshift.io/cluster -o jsonpath='{.spec.platformSpec.type}'

clusterPlatform, err = farUtils.GetClusterInfo(configClient)
clusterPlatformType = string(clusterPlatform.Status.PlatformStatus.Type) //infrastructure.Status.PlatformStatus.Type
clusterPlatformType = string(clusterPlatform.Status.PlatformStatus.Type)
if err != nil {
razo7 marked this conversation as resolved.
Show resolved Hide resolved
Fail("can't identify the cluster platform")
// fmt.Printf("can't identify the cluster platform")
}
fmt.Printf("\ncluster name: %s and PlatformType: %s \n", string(clusterPlatform.Name), clusterPlatformType)
})
Expand All @@ -74,8 +72,20 @@ var _ = Describe("FAR E2e", func() {
})

Context("fence agent - non-Dummy", func() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe a better name than non-Dummy ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was using the term dummy for FA which is dumb and does not act as a real FA in comparison to a real one who
supports some FA actions and actually communicate with a machine/node.

Is fence agent - fence_aws or fence_ipmilan sounds better to you?
Anyway will address this possible change in #20 and merge the current PR.

//testShareParam,testNodeParam := buildParameters(clusterPlatform, "status")
BeforeEach(func() {
var testNodeName string
nodes := &corev1.NodeList{}

Expect(k8sClient.List(context.Background(), nodes, &client.ListOptions{})).ToNot(HaveOccurred())
if len(nodes.Items) <= 1 {
Fail("there is one or less available nodes in the cluster")
}
//TODO: Randomize the node selection
// run FA on the first node - a master node
nodeObj := nodes.Items[nodeIndex]
testNodeName = nodeObj.Name
log.Info("Testing Node", "Node name", testNodeName)

if clusterPlatformType == "AWS" {
fenceAgent = fenceAgentAWS
By("running fence_aws")
Expand All @@ -86,28 +96,14 @@ var _ = Describe("FAR E2e", func() {
Skip("FAR haven't been tested on this kind of cluster (non AWS or BareMetal)")
}

testShareParam, err := buildSharedParameters(clusterPlatform, "status")
testShareParam, err := buildSharedParameters(clusterPlatform, fenceAgentAction)
if err != nil {
Fail("can't get shared information")
// fmt.Printf("can't get nodes' information- AWS instance ID")
}
testNodeParam, err := buildNodeParameters(clusterPlatformType)
if err != nil {
Fail("can't get node information")
// fmt.Printf("can't get nodes' information- AWS instance ID")
}
var testNodeName string
nodes := &corev1.NodeList{}

Expect(k8sClient.List(context.Background(), nodes, &client.ListOptions{})).ToNot(HaveOccurred())
if len(nodes.Items) <= 1 {
Fail("there is one or less available nodes in the cluster")
}
//TODO: Randomize the node selection
// run FA on the first node - a master node
nodeObj := nodes.Items[nodeIndex]
testNodeName = nodeObj.Name
log.Info("Testing Node", "Node name", testNodeName)

far = createFAR(testNodeName, fenceAgent, testShareParam, testNodeParam)
})
Expand All @@ -123,19 +119,42 @@ var _ = Describe("FAR E2e", func() {
Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(far), testFarCR)).To(Succeed(), "failed to get FAR CR")

By("checking the command has been executed successfully")
checkFarLogs("ON")
checkFarLogs(suceessStatusMessage)
})
})
})
})

// createFAR assigns the input to FenceAgentsRemediation object, creates CR, and returns the CR object
func createFAR(nodeName string, agent string, sharedParameters map[v1alpha1.ParameterName]string, nodeParameters map[v1alpha1.ParameterName]map[v1alpha1.NodeName]string) *v1alpha1.FenceAgentsRemediation {
far := &v1alpha1.FenceAgentsRemediation{
ObjectMeta: metav1.ObjectMeta{Name: nodeName, Namespace: testNamespace},
Spec: v1alpha1.FenceAgentsRemediationSpec{
Agent: agent,
SharedParameters: sharedParameters,
NodeParameters: nodeParameters,
},
}
ExpectWithOffset(1, k8sClient.Create(context.Background(), far)).ToNot(HaveOccurred())
return far
}

// deleteFAR deletes the CR with offset
func deleteFAR(far *v1alpha1.FenceAgentsRemediation) {
EventuallyWithOffset(1, func() error {
err := k8sClient.Delete(context.Background(), far)
if apiErrors.IsNotFound(err) {
return nil
}
return err
}, 2*time.Minute, 10*time.Second).ShouldNot(HaveOccurred(), "failed to delete far")
}

// buildSharedParameters returns a map key-value of shared parameters based on cluster platform type if it finds the credentials, otherwise an error
func buildSharedParameters(clusterPlatform *configv1.Infrastructure, action string) (map[v1alpha1.ParameterName]string, error) {
const (
secretAWS = "aws-cloud-credentials"
// https://github.com/openshift/release/blob/master/ci-operator/jobs/medik8s/fence-agents-remediation/medik8s-fence-agents-remediation-main-presubmits.yaml#L171

//AWS
// secretAWSOptional = "cluster-secrets-aws"
secretAWS = "aws-cloud-credentials"
secretKeyAWS = "aws_access_key_id"
secretValAWS = "aws_secret_access_key"

Expand All @@ -145,32 +164,30 @@ func buildSharedParameters(clusterPlatform *configv1.Infrastructure, action stri
secretKeyBM = "username"
secretValBM = "password"
)
var (
testShareParam map[v1alpha1.ParameterName]string
)
clusterPlatformType := string(clusterPlatform.Status.PlatformStatus.Type) //infrastructure.Status.PlatformStatus.Type
var testShareParam map[v1alpha1.ParameterName]string

// oc get Infrastructure.config.openshift.io/cluster -o jsonpath='{.status.platformStatus.type}'
clusterPlatformType := string(clusterPlatform.Status.PlatformStatus.Type)
if clusterPlatformType == "AWS" {
accessKey, secretKey, err := farUtils.GetCredientals(clientSet, secretAWS, secretKeyAWS, secretValAWS)
if err != nil {
fmt.Printf("can't get AWS credentials")
return nil, err
}

// command -> oc get Infrastructure.config.openshift.io/cluster -o jsonpath='{.status.platformStatus.aws.region}'
// oc get Infrastructure.config.openshift.io/cluster -o jsonpath='{.status.platformStatus.aws.region}'
regionAWS := string(clusterPlatform.Status.PlatformStatus.AWS.Region)

testShareParam = map[v1alpha1.ParameterName]string{
"--access-key": accessKey,
"--secret-key": secretKey,
"--region": regionAWS,
"--action": action,
"--verbose": "",
// "--verbose": "", // for verbose result
}

} else if clusterPlatformType == "BareMetal" {
// TODO : get ip from GetCredientals
// command -> oc get bmh -n openshift-machine-api ostest-master-0 -o jsonpath='{.spec.bmc.address}'
// oc get bmh -n openshift-machine-api ostest-master-0 -o jsonpath='{.spec.bmc.address}'
// then parse ip
username, password, err := farUtils.GetCredientals(clientSet, secretBMHExample, secretKeyBM, secretValBM)
if err != nil {
Expand All @@ -188,6 +205,7 @@ func buildSharedParameters(clusterPlatform *configv1.Infrastructure, action stri
return testShareParam, nil
}

// buildNodeParameters returns a map key-value of node parameters based on cluster platform type if it finds the node info list, otherwise an error
func buildNodeParameters(clusterPlatformType string) (map[v1alpha1.ParameterName]map[v1alpha1.NodeName]string, error) {

var (
Expand All @@ -207,44 +225,18 @@ func buildNodeParameters(clusterPlatformType string) (map[v1alpha1.ParameterName
nodeIdentifier = v1alpha1.ParameterName("--plug")

} else if clusterPlatformType == "BareMetal" {
nodeListParam, err = farUtils.GetBMNodeInfoList(machineClient)
nodeListParam, err = farUtils.GetBMHNodeInfoList(machineClient)
if err != nil {
// Fail("can't get nodes' information- AWS instance ID")
// Fail("can't get nodes' information- ports")
fmt.Printf("can't get nodes' information - ports")
return nil, err
}
nodeIdentifier = v1alpha1.ParameterName("--ipport")
}
testNodeParam = map[v1alpha1.ParameterName]map[v1alpha1.NodeName]string{nodeIdentifier: nodeListParam}

return testNodeParam, nil
}

// createFAR assigns the input to FenceAgentsRemediation object, creates CR, and returns the CR object
func createFAR(nodeName string, agent string, sharedParameters map[v1alpha1.ParameterName]string, nodeParameters map[v1alpha1.ParameterName]map[v1alpha1.NodeName]string) *v1alpha1.FenceAgentsRemediation {
far := &v1alpha1.FenceAgentsRemediation{
ObjectMeta: metav1.ObjectMeta{Name: nodeName, Namespace: testNamespace},
Spec: v1alpha1.FenceAgentsRemediationSpec{
Agent: agent,
SharedParameters: sharedParameters,
NodeParameters: nodeParameters,
},
}
ExpectWithOffset(1, k8sClient.Create(context.Background(), far)).ToNot(HaveOccurred())
return far
}

// deleteFAR deletes the CR with offset
func deleteFAR(far *v1alpha1.FenceAgentsRemediation) {
EventuallyWithOffset(1, func() error {
err := k8sClient.Delete(context.Background(), far)
if apiErrors.IsNotFound(err) {
return nil
}
return err
}, 2*time.Minute, 10*time.Second).ShouldNot(HaveOccurred(), "failed to delete far")
}

// checkFarLogs gets the FAR pod and checks whether it's logs have logString
func checkFarLogs(logString string) {
var pod *corev1.Pod
Expand Down
54 changes: 18 additions & 36 deletions test/e2e/utils/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
package utils

// Copy paste from https://github.com/hybrid-cloud-patterns/patterns-operator/blob/main/controllers/pattern_controller.go#L293-L313

import (
"context"
"fmt"

// "encoding/json"

"github.com/medik8s/fence-agents-remediation/api/v1alpha1"
configv1 "github.com/openshift/api/config/v1"
configclient "github.com/openshift/client-go/config/clientset/versioned"
Expand All @@ -17,43 +13,38 @@ import (
)

const (
machineNamespace = "openshift-machine-api"
machinesNamespace = "openshift-machine-api"
clusterPlatformName = "cluster"
)

// GetClusterInfo fetch the cluster's infrastructure object to identify it's type
func GetClusterInfo(config configclient.Interface) (*configv1.Infrastructure, error) {
// oc get Infrastructure.config.openshift.io/cluster

// clusterList, _ := config.ConfigV1().Infrastructures().List(context.Background(), metav1.ListOptions{})
// for _, cluster := range clusterList.Items {
// fmt.Printf("\ncluster name:%s and PlatformType: %s \n", string(cluster.Name), string(cluster.Status.PlatformStatus.Type))
// }
// Copy paste from https://github.com/hybrid-cloud-patterns/patterns-operator/blob/main/controllers/pattern_controller.go#L293-L313
// oc get Infrastructure.config.openshift.io/cluster -o jsonpath='{.metadata.name}'
// oc get Infrastructure.config.openshift.io/cluster -o jsonpath='{.spec.platformSpec.type}'

clusterInfra, err := config.ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
clusterInfra, err := config.ConfigV1().Infrastructures().Get(context.Background(), clusterPlatformName, metav1.GetOptions{})
if err != nil {
return nil, err
}
return clusterInfra, nil
}

// GetAWSCredientals searches for AWS secret, and then returns it decoded
// GetCredientals searches for AWS or BMH secret, and then returns it decoded
func GetCredientals(clientSet *kubernetes.Clientset, secretName, secretKey, secretVal string) (string, string, error) {
razo7 marked this conversation as resolved.
Show resolved Hide resolved
// oc get secrets -n openshift-machine-api aws-cloud-credentials -o jsonpath='{.data.aws_access_key_id}' | base64 -d
// oc get secrets -n openshift-machine-api aws-cloud-credentials -o jsonpath='{.data.aws_secret_access_key}' | base64 -d

secret, err := clientSet.CoreV1().Secrets(machineNamespace).Get(context.Background(), secretName, metav1.GetOptions{})
secret, err := clientSet.CoreV1().Secrets(machinesNamespace).Get(context.Background(), secretName, metav1.GetOptions{})
if err != nil {
return "", "", err
}
fmt.Printf("Key: %s Value: %s \n", secret.Data[secretKey], secret.Data[secretVal])
// Parse the AWS credentials - AWS access key and secret key
// var credentials v1beta1credential.OpenShiftCredentialProvider
// err = json.Unmarshal(awsSecret.Data["credentials"], &credentials)
// if err != nil {
// return "","", err
// }
// return credentials.Spec.SecretRef.KeyValues["access_key_id"], credentials.Spec.SecretRef.KeyValues["secret_access_key"], nil
// return keyID, keySecret, nil
if secretName == "aws-cloud-credentials" {
fmt.Printf("Key: %s & Value: %s \n", secret.Data[secretKey], secret.Data[secretVal])
clobrano marked this conversation as resolved.
Show resolved Hide resolved
} else {
fmt.Printf("Username: %s & Password: %s \n", secret.Data[secretKey], secret.Data[secretVal])
}

return string(secret.Data[secretKey]), string(secret.Data[secretVal]), nil
}

Expand All @@ -63,8 +54,9 @@ func GetAWSNodeInfoList(machineClient *machineclient.MachineV1beta1Client) (map[
// oc get machine -n openshift-machine-api MACHINE_NAME -o jsonpath='{.status.nodeRef.name}'

nodeList := make(map[v1alpha1.NodeName]string)

// Get the list of Machines in the openshift-machine-api namespace
machineList, err := machineClient.Machines("openshift-machine-api").List(context.Background(), metav1.ListOptions{})
machineList, err := machineClient.Machines(machinesNamespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
return nodeList, err
}
Expand All @@ -78,8 +70,8 @@ func GetAWSNodeInfoList(machineClient *machineclient.MachineV1beta1Client) (map[
return nodeList, nil
}

// GetNodeInfoList returns a list of the node names and their identification, e.g., AWS instance ID
func GetBMNodeInfoList(machineClient *machineclient.MachineV1beta1Client) (map[v1alpha1.NodeName]string, error) {
// GetBMHNodeInfoList returns a list of the node names and their identification, e.g., ports
func GetBMHNodeInfoList(machineClient *machineclient.MachineV1beta1Client) (map[v1alpha1.NodeName]string, error) {

//TODO: seacrch for BM and fetch ports

Expand All @@ -91,15 +83,5 @@ func GetBMNodeInfoList(machineClient *machineclient.MachineV1beta1Client) (map[v
"worker-1": "6234",
"worker-2": "6235",
}
// testNodeParam = map[v1alpha1.ParameterName]map[v1alpha1.NodeName]string{
// "--ipport": {
// "master-0": "6230",
// "master-1": "6231",
// "master-2": "6232",
// "worker-0": "6233",
// "worker-1": "6234",
// "worker-2": "6235",
// },
// }
return nodeList, nil
}