diff --git a/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java b/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java index ce34c48..ecffe82 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java +++ b/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java @@ -6,12 +6,16 @@ import io.fabric8.kubernetes.api.model.LabelSelector; import io.fabric8.kubernetes.api.model.LabelSelectorBuilder; +import io.fabric8.kubernetes.api.model.Namespace; +import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.odh.test.Environment; +import io.odh.test.OdhAnnotationsLabels; import io.odh.test.OdhConstants; import io.odh.test.TestSuite; import io.odh.test.install.BundleInstall; import io.odh.test.utils.DeploymentUtils; import io.odh.test.utils.UpgradeUtils; +import io.qameta.allure.Allure; import io.skodjob.annotations.Contact; import io.skodjob.annotations.Desc; import io.skodjob.annotations.Step; @@ -88,6 +92,17 @@ void testUpgradeBundle() throws IOException { String ntbNamespace = "test-odh-notebook-upgrade"; deployDsc(dsProjectName); + + Allure.step("Deploy notebook before upgrade"); + Namespace ns = new NamespaceBuilder() + .withNewMetadata() + .withName(ntbNamespace) + .addToLabels(OdhAnnotationsLabels.LABEL_DASHBOARD, "true") + .addToAnnotations(OdhAnnotationsLabels.ANNO_SERVICE_MESH, "false") + .endMetadata() + .build(); + KubeResourceManager.getInstance().createResourceWithoutWait(ns); + deployNotebook(ntbNamespace, ntbName); Map operatorSnapshot = DeploymentUtils.depSnapshot(baseBundle.getNamespace(), baseBundle.getDeploymentName()); diff --git a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java index 8623e45..98b0ca4 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java +++ b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java @@ -4,16 +4,22 @@ */ package io.odh.test.e2e.upgrade; +import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.LabelSelector; import io.fabric8.kubernetes.api.model.LabelSelectorBuilder; +import io.fabric8.kubernetes.api.model.Namespace; +import io.fabric8.kubernetes.api.model.NamespaceBuilder; +import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlan; import io.odh.test.Environment; +import io.odh.test.OdhAnnotationsLabels; import io.odh.test.OdhConstants; import io.odh.test.TestSuite; import io.odh.test.TestUtils; import io.odh.test.install.OlmInstall; import io.odh.test.utils.DeploymentUtils; import io.odh.test.utils.UpgradeUtils; +import io.qameta.allure.Allure; import io.skodjob.annotations.Contact; import io.skodjob.annotations.Desc; import io.skodjob.annotations.Step; @@ -23,13 +29,16 @@ import io.skodjob.testframe.resources.KubeResourceManager; import io.skodjob.testframe.utils.KubeUtils; import io.skodjob.testframe.utils.PodUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.kubeflow.v1.Notebook; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.time.Instant; +import java.util.List; import java.util.Map; @SuiteDoc( @@ -70,8 +79,9 @@ public class OlmUpgradeST extends UpgradeAbstract { ) @Test void testUpgradeOlm() throws IOException, InterruptedException { - String ntbName = "test-odh-notebook"; - String ntbNamespace = "test-odh-notebook-upgrade"; + final String ntbNamePre = "test-odh-notebook-pre"; + final String ntbNamePost = "test-odh-notebook-post"; + final String ntbNamespace = "test-odh-notebook-upgrade"; OlmInstall olmInstall = new OlmInstall(); olmInstall.setApproval("Manual"); @@ -90,13 +100,28 @@ void testUpgradeOlm() throws IOException, InterruptedException { // Deploy DSC deployDsc(DS_PROJECT_NAME); - deployNotebook(ntbNamespace, ntbName); - LabelSelector lblSelector = new LabelSelectorBuilder() - .withMatchLabels(Map.of("app", ntbName)) + Allure.step("Deploy notebook before upgrade"); + Namespace ns = new NamespaceBuilder() + .withNewMetadata() + .withName(ntbNamespace) + .addToLabels(OdhAnnotationsLabels.LABEL_DASHBOARD, "true") + .addToAnnotations(OdhAnnotationsLabels.ANNO_SERVICE_MESH, "false") + .endMetadata() .build(); + KubeResourceManager.getInstance().createResourceWithoutWait(ns); - PodUtils.waitForPodsReady(ntbNamespace, lblSelector, 1, true, () -> { }); + deployNotebook(ntbNamespace, ntbNamePre); + + LabelSelector lblSelectorPre = new LabelSelectorBuilder() + .withMatchLabels(Map.of("app", ntbNamePre)) + .build(); + + PodUtils.waitForPodsReady(ntbNamespace, lblSelectorPre, 1, true, () -> { }); + Notebook ntbResourcePre = KubeResourceManager.getKubeClient().getClient().resources(Notebook.class).inNamespace(ntbNamespace).withName(ntbNamePre).get(); + List ntbPodsPre = KubeResourceManager.getKubeClient().getClient().pods() + .inNamespace(ntbNamespace).withLabelSelector(lblSelectorPre).list().getItems(); + Assertions.assertEquals(1, ntbPodsPre.size()); LOGGER.info("Upgrade to next available version in OLM catalog"); // Approve upgrade to newer version @@ -113,8 +138,38 @@ void testUpgradeOlm() throws IOException, InterruptedException { Instant operatorLogCheckTimestamp = Instant.now(); // Verify that NTB pods are stable - PodUtils.waitForPodsReady(ntbNamespace, lblSelector, 1, true, () -> { }); + PodUtils.waitForPodsReady(ntbNamespace, lblSelectorPre, 1, true, () -> { }); // Check logs in operator pod UpgradeUtils.deploymentLogIsErrorEmpty(olmInstall.getNamespace(), olmInstall.getDeploymentName(), operatorLogCheckTimestamp); + + // RHOAIENG-10827: creating new notebooks after upgrade restarted existing notebooks + Allure.step("Deploy another notebook after upgrade"); + deployNotebook(ntbNamespace, ntbNamePost); + + LabelSelector lblSelectorPost = new LabelSelectorBuilder() + .withMatchLabels(Map.of("app", ntbNamePost)) + .build(); + PodUtils.waitForPodsReady(ntbNamespace, lblSelectorPost, 1, true, () -> { }); + + // Verify that old notebook pod is running + PodUtils.waitForPodsReady(ntbNamespace, lblSelectorPre, 1, true, () -> { }); + // and + List ntbPodsPreAfter = KubeResourceManager.getKubeClient().getClient().pods() + .inNamespace(ntbNamespace).withLabelSelector(lblSelectorPre).list().getItems(); + Assertions.assertEquals(1, ntbPodsPreAfter.size()); + // has not been restarted + for (ContainerStatus containerStatus : ntbPodsPreAfter.get(0).getStatus().getContainerStatuses()) { + Assertions.assertEquals(0, containerStatus.getRestartCount(), containerStatus.toString()); + } + // nor recreated + Assertions.assertEquals(ntbPodsPre.get(0).getMetadata().getUid(), ntbPodsPreAfter.get(0).getMetadata().getUid()); + // nor modified in any way + Assertions.assertEquals(ntbPodsPre.get(0).getMetadata().getResourceVersion(), ntbPodsPreAfter.get(0).getMetadata().getResourceVersion()); + + // Verify that old notebook resource spec has not been modified + Notebook ntbResourcePreAfter = KubeResourceManager.getKubeClient().getClient().resources(Notebook.class).inNamespace(ntbNamespace).withName(ntbNamePre).get(); + Assertions.assertEquals(ntbResourcePre.getMetadata().getUid(), ntbResourcePreAfter.getMetadata().getUid()); + Assertions.assertEquals(ntbResourcePre.getSpec(), ntbResourcePreAfter.getSpec()); + Assertions.assertEquals(ntbResourcePre.getMetadata().getResourceVersion(), ntbResourcePreAfter.getMetadata().getResourceVersion()); } } diff --git a/src/test/java/io/odh/test/e2e/upgrade/UpgradeAbstract.java b/src/test/java/io/odh/test/e2e/upgrade/UpgradeAbstract.java index 48e156e..4485b44 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/UpgradeAbstract.java +++ b/src/test/java/io/odh/test/e2e/upgrade/UpgradeAbstract.java @@ -4,8 +4,6 @@ */ package io.odh.test.e2e.upgrade; -import io.fabric8.kubernetes.api.model.Namespace; -import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; import io.fabric8.kubernetes.api.model.PersistentVolumeClaimBuilder; import io.fabric8.kubernetes.api.model.Quantity; @@ -89,15 +87,6 @@ protected void deployDsc(String name) { KubeResourceManager.getInstance().createResourceWithWait(dsc); } public void deployNotebook(String namespace, String name) throws IOException { - Namespace ns = new NamespaceBuilder() - .withNewMetadata() - .withName(namespace) - .addToLabels(OdhAnnotationsLabels.LABEL_DASHBOARD, "true") - .addToAnnotations(OdhAnnotationsLabels.ANNO_SERVICE_MESH, "false") - .endMetadata() - .build(); - KubeResourceManager.getInstance().createResourceWithoutWait(ns); - PersistentVolumeClaim pvc = new PersistentVolumeClaimBuilder() .withNewMetadata() .withName(name)