Skip to content

Commit

Permalink
feat(upgrade): check that old notebook is not restarted after upgrade
Browse files Browse the repository at this point in the history
  • Loading branch information
@jiridanek @jstourac
jiridanek authored and jstourac committed Oct 18, 2024
1 parent 1330deb commit d8b94c2
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 18 deletions.
15 changes: 15 additions & 0 deletions src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,16 @@

import io.fabric8.kubernetes.api.model.LabelSelector;
import io.fabric8.kubernetes.api.model.LabelSelectorBuilder;
import io.fabric8.kubernetes.api.model.Namespace;
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.odh.test.Environment;
import io.odh.test.OdhAnnotationsLabels;
import io.odh.test.OdhConstants;
import io.odh.test.TestSuite;
import io.odh.test.install.BundleInstall;
import io.odh.test.utils.DeploymentUtils;
import io.odh.test.utils.UpgradeUtils;
import io.qameta.allure.Allure;
import io.skodjob.annotations.Contact;
import io.skodjob.annotations.Desc;
import io.skodjob.annotations.Step;
Expand Down Expand Up @@ -88,6 +92,17 @@ void testUpgradeBundle() throws IOException {
String ntbNamespace = "test-odh-notebook-upgrade";

deployDsc(dsProjectName);

Allure.step("Deploy notebook before upgrade");
Namespace ns = new NamespaceBuilder()
.withNewMetadata()
.withName(ntbNamespace)
.addToLabels(OdhAnnotationsLabels.LABEL_DASHBOARD, "true")
.addToAnnotations(OdhAnnotationsLabels.ANNO_SERVICE_MESH, "false")
.endMetadata()
.build();
KubeResourceManager.getInstance().createResourceWithoutWait(ns);

deployNotebook(ntbNamespace, ntbName);

Map<String, String> operatorSnapshot = DeploymentUtils.depSnapshot(baseBundle.getNamespace(), baseBundle.getDeploymentName());
Expand Down
69 changes: 62 additions & 7 deletions src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,22 @@
*/
package io.odh.test.e2e.upgrade;

import io.fabric8.kubernetes.api.model.ContainerStatus;
import io.fabric8.kubernetes.api.model.LabelSelector;
import io.fabric8.kubernetes.api.model.LabelSelectorBuilder;
import io.fabric8.kubernetes.api.model.Namespace;
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlan;
import io.odh.test.Environment;
import io.odh.test.OdhAnnotationsLabels;
import io.odh.test.OdhConstants;
import io.odh.test.TestSuite;
import io.odh.test.TestUtils;
import io.odh.test.install.OlmInstall;
import io.odh.test.utils.DeploymentUtils;
import io.odh.test.utils.UpgradeUtils;
import io.qameta.allure.Allure;
import io.skodjob.annotations.Contact;
import io.skodjob.annotations.Desc;
import io.skodjob.annotations.Step;
Expand All @@ -23,13 +29,16 @@
import io.skodjob.testframe.resources.KubeResourceManager;
import io.skodjob.testframe.utils.KubeUtils;
import io.skodjob.testframe.utils.PodUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.kubeflow.v1.Notebook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Map;

@SuiteDoc(
Expand Down Expand Up @@ -70,8 +79,9 @@ public class OlmUpgradeST extends UpgradeAbstract {
)
@Test
void testUpgradeOlm() throws IOException, InterruptedException {
String ntbName = "test-odh-notebook";
String ntbNamespace = "test-odh-notebook-upgrade";
final String ntbNamePre = "test-odh-notebook-pre";
final String ntbNamePost = "test-odh-notebook-post";
final String ntbNamespace = "test-odh-notebook-upgrade";

OlmInstall olmInstall = new OlmInstall();
olmInstall.setApproval("Manual");
Expand All @@ -90,13 +100,28 @@ void testUpgradeOlm() throws IOException, InterruptedException {

// Deploy DSC
deployDsc(DS_PROJECT_NAME);
deployNotebook(ntbNamespace, ntbName);

LabelSelector lblSelector = new LabelSelectorBuilder()
.withMatchLabels(Map.of("app", ntbName))
Allure.step("Deploy notebook before upgrade");
Namespace ns = new NamespaceBuilder()
.withNewMetadata()
.withName(ntbNamespace)
.addToLabels(OdhAnnotationsLabels.LABEL_DASHBOARD, "true")
.addToAnnotations(OdhAnnotationsLabels.ANNO_SERVICE_MESH, "false")
.endMetadata()
.build();
KubeResourceManager.getInstance().createResourceWithoutWait(ns);

PodUtils.waitForPodsReady(ntbNamespace, lblSelector, 1, true, () -> { });
deployNotebook(ntbNamespace, ntbNamePre);

LabelSelector lblSelectorPre = new LabelSelectorBuilder()
.withMatchLabels(Map.of("app", ntbNamePre))
.build();

PodUtils.waitForPodsReady(ntbNamespace, lblSelectorPre, 1, true, () -> { });
Notebook ntbResourcePre = KubeResourceManager.getKubeClient().getClient().resources(Notebook.class).inNamespace(ntbNamespace).withName(ntbNamePre).get();
List<Pod> ntbPodsPre = KubeResourceManager.getKubeClient().getClient().pods()
.inNamespace(ntbNamespace).withLabelSelector(lblSelectorPre).list().getItems();
Assertions.assertEquals(1, ntbPodsPre.size());

LOGGER.info("Upgrade to next available version in OLM catalog");
// Approve upgrade to newer version
Expand All @@ -113,8 +138,38 @@ void testUpgradeOlm() throws IOException, InterruptedException {
Instant operatorLogCheckTimestamp = Instant.now();

// Verify that NTB pods are stable
PodUtils.waitForPodsReady(ntbNamespace, lblSelector, 1, true, () -> { });
PodUtils.waitForPodsReady(ntbNamespace, lblSelectorPre, 1, true, () -> { });
// Check logs in operator pod
UpgradeUtils.deploymentLogIsErrorEmpty(olmInstall.getNamespace(), olmInstall.getDeploymentName(), operatorLogCheckTimestamp);

// RHOAIENG-10827: creating new notebooks after upgrade restarted existing notebooks
Allure.step("Deploy another notebook after upgrade");
deployNotebook(ntbNamespace, ntbNamePost);

LabelSelector lblSelectorPost = new LabelSelectorBuilder()
.withMatchLabels(Map.of("app", ntbNamePost))
.build();
PodUtils.waitForPodsReady(ntbNamespace, lblSelectorPost, 1, true, () -> { });

// Verify that old notebook pod is running
PodUtils.waitForPodsReady(ntbNamespace, lblSelectorPre, 1, true, () -> { });
// and
List<Pod> ntbPodsPreAfter = KubeResourceManager.getKubeClient().getClient().pods()
.inNamespace(ntbNamespace).withLabelSelector(lblSelectorPre).list().getItems();
Assertions.assertEquals(1, ntbPodsPreAfter.size());
// has not been restarted
for (ContainerStatus containerStatus : ntbPodsPreAfter.get(0).getStatus().getContainerStatuses()) {
Assertions.assertEquals(0, containerStatus.getRestartCount(), containerStatus.toString());
}
// nor recreated
Assertions.assertEquals(ntbPodsPre.get(0).getMetadata().getUid(), ntbPodsPreAfter.get(0).getMetadata().getUid());
// nor modified in any way
Assertions.assertEquals(ntbPodsPre.get(0).getMetadata().getResourceVersion(), ntbPodsPreAfter.get(0).getMetadata().getResourceVersion());

// Verify that old notebook resource spec has not been modified
Notebook ntbResourcePreAfter = KubeResourceManager.getKubeClient().getClient().resources(Notebook.class).inNamespace(ntbNamespace).withName(ntbNamePre).get();
Assertions.assertEquals(ntbResourcePre.getMetadata().getUid(), ntbResourcePreAfter.getMetadata().getUid());
Assertions.assertEquals(ntbResourcePre.getSpec(), ntbResourcePreAfter.getSpec());
Assertions.assertEquals(ntbResourcePre.getMetadata().getResourceVersion(), ntbResourcePreAfter.getMetadata().getResourceVersion());
}
}
11 changes: 0 additions & 11 deletions src/test/java/io/odh/test/e2e/upgrade/UpgradeAbstract.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
*/
package io.odh.test.e2e.upgrade;

import io.fabric8.kubernetes.api.model.Namespace;
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaimBuilder;
import io.fabric8.kubernetes.api.model.Quantity;
Expand Down Expand Up @@ -89,15 +87,6 @@ protected void deployDsc(String name) {
KubeResourceManager.getInstance().createResourceWithWait(dsc);
}
public void deployNotebook(String namespace, String name) throws IOException {
Namespace ns = new NamespaceBuilder()
.withNewMetadata()
.withName(namespace)
.addToLabels(OdhAnnotationsLabels.LABEL_DASHBOARD, "true")
.addToAnnotations(OdhAnnotationsLabels.ANNO_SERVICE_MESH, "false")
.endMetadata()
.build();
KubeResourceManager.getInstance().createResourceWithoutWait(ns);

PersistentVolumeClaim pvc = new PersistentVolumeClaimBuilder()
.withNewMetadata()
.withName(name)
Expand Down

0 comments on commit d8b94c2

Please sign in to comment.