diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh index 9bd9ac2789..b2accf3b8f 100755 --- a/hack/ci-e2e.sh +++ b/hack/ci-e2e.sh @@ -115,7 +115,7 @@ IMAGE_DIR="${REPO_ROOT}/test/e2e/images" ## Download and run image server mkdir -p "${IMAGE_DIR}" pushd "${IMAGE_DIR}" -wget --quiet "https://download.cirros-cloud.net/${CIRROS_VERSION}/${IMAGE_FILE}" +wget --quiet https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}" popd docker run --name image-server-e2e -d \ diff --git a/hack/clean-e2e.sh b/hack/clean-e2e.sh index 75c4a597e4..06d29ceb09 100755 --- a/hack/clean-e2e.sh +++ b/hack/clean-e2e.sh @@ -8,10 +8,10 @@ docker rm -f vbmc docker rm -f image-server-e2e docker rm -f sushy-tools virsh -c qemu:///system destroy --domain bmo-e2e-0 -virsh -c qemu:///system undefine --domain bmo-e2e-0 --remove-all-storage +virsh -c qemu:///system undefine --domain bmo-e2e-0 --nvram --remove-all-storage virsh -c qemu:///system net-destroy baremetal-e2e virsh -c qemu:///system net-undefine baremetal-e2e -rm -rfv "${REPO_ROOT}/test/e2e/_artifacts" -rm -rfv "${REPO_ROOT}/artifacts.tar.gz" -rm -rfv "${REPO_ROOT}/test/e2e/images" +rm -rf "${REPO_ROOT}/test/e2e/_artifacts" +rm -rf "${REPO_ROOT}/artifacts.tar.gz" +rm -rf "${REPO_ROOT}/test/e2e/images" diff --git a/hack/e2e/net.xml b/hack/e2e/net.xml index f64fd04f9f..54a2740bfb 100644 --- a/hack/e2e/net.xml +++ b/hack/e2e/net.xml @@ -9,7 +9,8 @@ - + + diff --git a/test/e2e/common.go b/test/e2e/common.go index 234c431379..0c84e16ec4 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "golang.org/x/crypto/ssh" + "github.com/pkg/errors" "gopkg.in/yaml.v2" @@ -18,6 +20,8 @@ import ( metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" + capm3_e2e "github.com/metal3-io/cluster-api-provider-metal3/test/e2e" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/cluster-api/test/framework" @@ -221,3 +225,61 @@ func buildKustomizeManifest(source string) ([]byte, error) { } return resources.AsYaml() } + +func executeSSHCommand(client *ssh.Client, command string) (string, error) { + session, err := client.NewSession() + if err != nil { + return "", fmt.Errorf("failed to create SSH session: %v", err) + } + defer session.Close() + + output, err := session.CombinedOutput(command) + if err != nil { + return "", fmt.Errorf("failed to execute command '%s': %v", command, err) + } + + return string(output), nil +} + +// ParseFileSystemOutput parses the given filesystem output string and checks if booted from disk +func ParseFileSystemOutput(output string) bool { + lines := strings.Split(output, "\n") + + for _, line := range lines[1:] { // Skip header line + if line == "" { + continue + } + + fields := strings.Fields(line) + if len(fields) < 6 { + continue // Skip malformed lines + } + + if fields[5] == "/" && !strings.Contains(fields[0], "tmpfs") { + return true // Found a non-tmpfs root filesystem + } + } + + return false +} + +func IsBootedFromDisk(client *ssh.Client) (bool, error) { + cmd := "df -h" + output, err := executeSSHCommand(client, cmd) + if err != nil { + return false, fmt.Errorf("error executing 'df -h': %w", err) + } + + isDisk := ParseFileSystemOutput(output) + if isDisk { + capm3_e2e.Logf("System is booted from a disk.") + } else { + capm3_e2e.Logf("System is booted from a live ISO.") + } + + return isDisk, nil +} + +func StringPtr(s string) *string { + return &s +} diff --git a/test/e2e/config/fixture.yaml b/test/e2e/config/fixture.yaml index 8d5d65d8cf..3fe47ecaa7 100644 --- a/test/e2e/config/fixture.yaml +++ b/test/e2e/config/fixture.yaml @@ -25,6 +25,7 @@ variables: IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906" CERT_MANAGER_VERSION: "v1.13.0" + SSH_CHECK_PROVISIONED: "false" intervals: inspection/wait-unmanaged: ["1m", "10ms"] diff --git a/test/e2e/config/ironic.yaml b/test/e2e/config/ironic.yaml index 7c930b76f0..a18776de09 100644 --- a/test/e2e/config/ironic.yaml +++ b/test/e2e/config/ironic.yaml @@ -19,7 +19,7 @@ variables: DEPLOY_CERT_MANAGER: "true" BMO_KUSTOMIZATION: "../../config/overlays/e2e" IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e" - EXPECTED_HOST_NAME: "localhost.localdomain" + EXPECTED_HOST_NAME: "bmo-e2e-0" # Test credentials. The tests will create a BMH with these. BMC_USER: admin BMC_PASSWORD: password @@ -32,6 +32,7 @@ variables: IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906" CERT_MANAGER_VERSION: "v1.13.0" + SSH_CHECK_PROVISIONED: "true" intervals: inspection/wait-unmanaged: ["1m", "5s"] @@ -49,3 +50,4 @@ intervals: default/wait-deprovisioning: ["1m", "10ms"] default/wait-deleted: ["20s", "10ms"] default/wait-secret-deletion: ["1m", "1s"] + default/wait-connect-ssh: ["2m", "10s"] diff --git a/test/e2e/external_inspection_test.go b/test/e2e/external_inspection_test.go index 7315a005c5..6295d7583e 100644 --- a/test/e2e/external_inspection_test.go +++ b/test/e2e/external_inspection_test.go @@ -133,7 +133,7 @@ const hardwareDetails = ` "version": "1.15.0-1" } }, - "hostname": "localhost.localdomain", + "hostname": "bmo-e2e-0", "nics": [ { "ip": "192.168.222.122", diff --git a/test/e2e/live_iso_test.go b/test/e2e/live_iso_test.go new file mode 100644 index 0000000000..4340210cc1 --- /dev/null +++ b/test/e2e/live_iso_test.go @@ -0,0 +1,163 @@ +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/crypto/ssh" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/patch" + + metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" + + capm3_e2e "github.com/metal3-io/cluster-api-provider-metal3/test/e2e" +) + +var _ = Describe("Live-ISO", func() { + var ( + specName = "live-iso-ops" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + bmcUser string + bmcPassword string + bmcAddress string + bootMacAddress string + ) + + BeforeEach(func() { + bmcUser = e2eConfig.GetVariable("BMC_USER") + bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD") + bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS") + bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS") + + namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + Creator: clusterProxy.GetClient(), + ClientSet: clusterProxy.GetClientSet(), + Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), + LogFolder: artifactFolder, + }) + }) + + It("should provision a BMH with live ISO and then deprovision it", func() { + By("Creating a secret with BMH credentials") + bmcCredentials := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bmc-credentials", + Namespace: namespace.Name, + }, + StringData: map[string]string{ + "username": bmcUser, + "password": bmcPassword, + }, + } + err := clusterProxy.GetClient().Create(ctx, &bmcCredentials) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a BMH with inspection disabled and hardware details added") + bmh := metal3api.BareMetalHost{ + ObjectMeta: metav1.ObjectMeta{ + Name: specName, + Namespace: namespace.Name, + Annotations: map[string]string{ + metal3api.InspectAnnotationPrefix: "disabled", + metal3api.HardwareDetailsAnnotation: hardwareDetails, + }, + }, + Spec: metal3api.BareMetalHostSpec{ + Online: true, + BMC: metal3api.BMCDetails{ + Address: bmcAddress, + CredentialsName: "bmc-credentials", + }, + Image: &metal3api.Image{ + URL: e2eConfig.GetVariable("IMAGE_URL"), + DiskFormat: StringPtr("live-iso"), + }, + BootMode: metal3api.Legacy, + BootMACAddress: bootMacAddress, + AutomatedCleaningMode: "disabled", + RootDeviceHints: &metal3api.RootDeviceHints{ + DeviceName: "/dev/vda", + }, + }, + } + err = clusterProxy.GetClient().Create(ctx, &bmh) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the BMH to be in provisioning state") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateProvisioning, + }, e2eConfig.GetIntervals(specName, "wait-provisioning")...) + + By("Waiting for the BMH to become provisioned") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateProvisioned, + }, e2eConfig.GetIntervals(specName, "wait-provisioned")...) + + By("Verifying the node booted from live ISO image") + // This is to bypass the fixture test only + if e2eConfig.GetVariable("SSH_CHECK_PROVISIONED") == "true" { + + // Set up SSH client configuration + config := &ssh.ClientConfig{ + User: "cirros", + Auth: []ssh.AuthMethod{ + ssh.Password("gocubsgo"), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), // #nosec G106 + } + + // Establish an SSH connection + var client *ssh.Client + + Eventually(func() error { + client, err = ssh.Dial("tcp", "192.168.222.122:22", config) + return err + }, e2eConfig.GetIntervals(specName, "wait-connect-ssh")...).Should(Succeed(), "Failed to establish SSH connection") + + defer func() { + if client != nil { + client.Close() + } + }() + isDisk, err := IsBootedFromDisk(client) + Expect(err).NotTo(HaveOccurred(), "Error in verifying boot mode") + Expect(isDisk).To(Equal(false), "Error booting from disk when live ISO is expected") + } else { + capm3_e2e.Logf("Fixture system is booted from a live ISO.") + } + + By("Triggering the deprovisioning of the BMH") + helper, err := patch.NewHelper(&bmh, clusterProxy.GetClient()) + Expect(err).NotTo(HaveOccurred()) + bmh.Spec.Image = nil + Expect(helper.Patch(ctx, &bmh)).To(Succeed()) + + By("Waiting for the BMH to be in deprovisioning state") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateDeprovisioning, + }, e2eConfig.GetIntervals(specName, "wait-deprovisioning")...) + + By("Waiting for the BMH to become available again") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateAvailable, + }, e2eConfig.GetIntervals(specName, "wait-available")...) + }) + + AfterEach(func() { + cleanup(ctx, clusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + }) +}) diff --git a/test/e2e/re_inspection_test.go b/test/e2e/re_inspection_test.go index 42e0739114..6c93f8c54c 100644 --- a/test/e2e/re_inspection_test.go +++ b/test/e2e/re_inspection_test.go @@ -64,7 +64,7 @@ var _ = Describe("Re-Inspection", func() { Expect(err).NotTo(HaveOccurred()) By("creating a BMH with inspection disabled and hardware details added with wrong HostName") - newHardwareDetails := strings.Replace(hardwareDetails, "localhost.localdomain", wrongHostName, 1) + newHardwareDetails := strings.Replace(hardwareDetails, "bmo-e2e-0", wrongHostName, 1) bmh := metal3api.BareMetalHost{ ObjectMeta: metav1.ObjectMeta{ Name: specName + "-reinspect", diff --git a/test/go.mod b/test/go.mod index d27866e3a4..3249f27d40 100644 --- a/test/go.mod +++ b/test/go.mod @@ -9,6 +9,7 @@ require ( github.com/onsi/ginkgo/v2 v2.13.0 github.com/onsi/gomega v1.29.0 github.com/pkg/errors v0.9.1 + golang.org/x/crypto v0.14.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.27.7 k8s.io/apimachinery v0.27.7 @@ -108,7 +109,6 @@ require ( github.com/valyala/fastjson v1.6.4 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.17.0 // indirect