From 1d8c6987daa66d9f8b11cadc8ec1be22d763f4c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sat, 9 Mar 2024 11:29:37 +0100 Subject: [PATCH 1/2] Update default OLM version for downstream install --- src/main/java/io/odh/test/OdhConstants.java | 6 +-- src/main/java/io/odh/test/utils/CsvUtils.java | 40 +++++++++++++++++++ .../e2e/continuous/DataScienceClusterST.java | 14 ++++++- .../e2e/standard/DataScienceClusterST.java | 14 ++++++- 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 src/main/java/io/odh/test/utils/CsvUtils.java diff --git a/src/main/java/io/odh/test/OdhConstants.java b/src/main/java/io/odh/test/OdhConstants.java index f34ddc6d..6914701b 100644 --- a/src/main/java/io/odh/test/OdhConstants.java +++ b/src/main/java/io/odh/test/OdhConstants.java @@ -49,9 +49,9 @@ private OdhConstants() { } private static final String RHOAI_OLM_OPERATOR_DEPLOYMENT_NAME = "rhods-operator"; private static final String RHOAI_OLM_SOURCE_NAME = "redhat-operators"; private static final String RHOAI_OLM_APP_BUNDLE_PREFIX = "rhods-operator"; - private static final String RHOAI_OLM_OPERATOR_CHANNEL = "stable"; - private static final String RHOAI_OLM_OPERATOR_VERSION = "2.6.0"; - private static final String RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION = "2.5.0"; + private static final String RHOAI_OLM_OPERATOR_CHANNEL = "fast"; + private static final String RHOAI_OLM_OPERATOR_VERSION = "2.7.0"; + private static final String RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION = "2.6.0"; private static final String RHOAI_MONITORING_NAMESPACE = "redhat-ods-monitoring"; // Public part diff --git a/src/main/java/io/odh/test/utils/CsvUtils.java b/src/main/java/io/odh/test/utils/CsvUtils.java new file mode 100644 index 00000000..524d043a --- /dev/null +++ b/src/main/java/io/odh/test/utils/CsvUtils.java @@ -0,0 +1,40 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ + +package io.odh.test.utils; + +import io.fabric8.openshift.api.model.operatorhub.v1alpha1.ClusterServiceVersion; +import io.fabric8.openshift.client.OpenShiftClient; +import io.odh.test.OdhConstants; +import io.odh.test.framework.manager.ResourceManager; +import org.hamcrest.Matchers; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; + +public class CsvUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(NamespaceUtils.class); + + public static @Nullable String getOperatorVersionFromCsv() { + String name = OdhConstants.OLM_OPERATOR_NAME; + String namespace = OdhConstants.OLM_OPERATOR_NAMESPACE; + OpenShiftClient client = (OpenShiftClient) ResourceManager.getKubeClient().getClient(); + List csvs = client.resources(ClusterServiceVersion.class) + .inNamespace(namespace) + .withLabel("operators.coreos.com/" + name + "." + namespace, "") + .list().getItems(); + LOGGER.debug("Found {} datascience operator CSVs", csvs.size()); + if (csvs.isEmpty()) { + return ""; + } + assertThat(csvs, Matchers.hasSize(1)); + return csvs.get(0).getSpec().getVersion(); + } +} diff --git a/src/test/java/io/odh/test/e2e/continuous/DataScienceClusterST.java b/src/test/java/io/odh/test/e2e/continuous/DataScienceClusterST.java index f2bbe35f..9e23f2b5 100644 --- a/src/test/java/io/odh/test/e2e/continuous/DataScienceClusterST.java +++ b/src/test/java/io/odh/test/e2e/continuous/DataScienceClusterST.java @@ -7,12 +7,15 @@ import io.fabric8.kubernetes.api.model.KubernetesResourceList; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.Resource; +import io.odh.test.Environment; import io.odh.test.OdhConstants; import io.odh.test.TestSuite; import io.odh.test.e2e.Abstract; import io.odh.test.framework.manager.ResourceManager; import io.odh.test.framework.manager.resources.DataScienceClusterResource; +import io.odh.test.install.InstallTypes; import io.odh.test.platform.KubeUtils; +import io.odh.test.utils.CsvUtils; import io.opendatahub.datasciencecluster.v1.DataScienceCluster; import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Codeflare; import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Dashboard; @@ -27,6 +30,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import java.util.Objects; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -56,7 +61,14 @@ void checkDataScienceClusterExists() { assertEquals(Ray.ManagementState.MANAGED, cluster.getSpec().getComponents().getRay().getManagementState()); assertEquals(Modelmeshserving.ManagementState.MANAGED, cluster.getSpec().getComponents().getModelmeshserving().getManagementState()); assertEquals(Datasciencepipelines.ManagementState.MANAGED, cluster.getSpec().getComponents().getDatasciencepipelines().getManagementState()); - assertEquals(Kueue.ManagementState.MANAGED, cluster.getSpec().getComponents().getKueue().getManagementState()); + if (!Environment.PRODUCT.equals(Environment.PRODUCT_DEFAULT) + && Environment.OPERATOR_INSTALL_TYPE.equalsIgnoreCase(InstallTypes.OLM.toString()) + && Objects.requireNonNull(CsvUtils.getOperatorVersionFromCsv()).equals("2.7.0")) { + // https://issues.redhat.com/browse/RHOAIENG-3234 Remove Kueue from RHOAI 2.7 + assertNull(cluster.getSpec().getComponents().getKueue()); + } else { + assertEquals(Kueue.ManagementState.MANAGED, cluster.getSpec().getComponents().getKueue().getManagementState()); + } assertEquals(Workbenches.ManagementState.MANAGED, cluster.getSpec().getComponents().getWorkbenches().getManagementState()); } diff --git a/src/test/java/io/odh/test/e2e/standard/DataScienceClusterST.java b/src/test/java/io/odh/test/e2e/standard/DataScienceClusterST.java index d41620f6..2a3fe201 100644 --- a/src/test/java/io/odh/test/e2e/standard/DataScienceClusterST.java +++ b/src/test/java/io/odh/test/e2e/standard/DataScienceClusterST.java @@ -8,6 +8,8 @@ import io.odh.test.TestSuite; import io.odh.test.framework.manager.ResourceManager; import io.odh.test.framework.manager.resources.DataScienceClusterResource; +import io.odh.test.install.InstallTypes; +import io.odh.test.utils.CsvUtils; import io.odh.test.utils.DscUtils; import io.opendatahub.datasciencecluster.v1.DataScienceCluster; import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Codeflare; @@ -29,7 +31,10 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable; +import java.util.Objects; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; @SuiteDoc( description = @Desc("Verifies simple setup of ODH by spin-up operator, setup DSCI, and setup DSC."), @@ -87,6 +92,13 @@ void createDataScienceCluster() { assertEquals(Workbenches.ManagementState.MANAGED, cluster.getSpec().getComponents().getWorkbenches().getManagementState()); assertEquals(Modelmeshserving.ManagementState.MANAGED, cluster.getSpec().getComponents().getModelmeshserving().getManagementState()); assertEquals(Ray.ManagementState.MANAGED, cluster.getSpec().getComponents().getRay().getManagementState()); - assertEquals(Kueue.ManagementState.MANAGED, cluster.getSpec().getComponents().getKueue().getManagementState()); + if (!Environment.PRODUCT.equals(Environment.PRODUCT_DEFAULT) + && Environment.OPERATOR_INSTALL_TYPE.equalsIgnoreCase(InstallTypes.OLM.toString()) + && Objects.requireNonNull(CsvUtils.getOperatorVersionFromCsv()).equals("2.7.0")) { + // https://issues.redhat.com/browse/RHOAIENG-3234 Remove Kueue from RHOAI 2.7 + assertNull(cluster.getSpec().getComponents().getKueue()); + } else { + assertEquals(Kueue.ManagementState.MANAGED, cluster.getSpec().getComponents().getKueue().getManagementState()); + } } } From 089672682cb6cba0b0f921c9ec7a6288b0744ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sat, 9 Mar 2024 11:28:16 +0100 Subject: [PATCH 2/2] Wait for DataScienceCluster to finish reconciling --- src/main/java/io/odh/test/OdhConstants.java | 2 ++ .../resources/DataScienceClusterResource.java | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/main/java/io/odh/test/OdhConstants.java b/src/main/java/io/odh/test/OdhConstants.java index 6914701b..95102af8 100644 --- a/src/main/java/io/odh/test/OdhConstants.java +++ b/src/main/java/io/odh/test/OdhConstants.java @@ -55,6 +55,8 @@ private OdhConstants() { } private static final String RHOAI_MONITORING_NAMESPACE = "redhat-ods-monitoring"; // Public part + public static final String DSC_CREATION_SUCCESSFUL_EVENT_NAME = "DataScienceClusterCreationSuccessful"; + public static final String CODEFLARE_DEPLOYMENT_NAME = "codeflare-operator-manager"; public static final String DS_PIPELINES_OPERATOR = "data-science-pipelines-operator-controller-manager"; public static final String ETCD = "etcd"; diff --git a/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java b/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java index b310eeba..56b3c373 100644 --- a/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java +++ b/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java @@ -5,6 +5,8 @@ package io.odh.test.framework.manager.resources; import io.fabric8.kubernetes.api.model.KubernetesResourceList; +import io.fabric8.kubernetes.api.model.events.v1.Event; +import io.fabric8.kubernetes.client.dsl.EventingAPIGroupDSL; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.Resource; import io.odh.test.OdhConstants; @@ -18,6 +20,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; +import java.util.Objects; + public class DataScienceClusterResource implements ResourceType { private static final Logger LOGGER = LoggerFactory.getLogger(DataScienceClusterResource.class); @@ -105,6 +110,25 @@ public boolean waitForReadiness(DataScienceCluster resource) { // dscReady = dscReady && pipelinesStatus.equals("True"); // } + // Check that DSC reconciliation has been successfully finalized + // https://github.com/red-hat-data-services/rhods-operator/blob/rhoai-2.8/controllers/datasciencecluster/datasciencecluster_controller.go#L257 + + // Wait for ReconcileComplete condition (for the whole DSC) + String reconcileStatus = KubeUtils.getDscConditionByType(dsc.getStatus().getConditions(), "ReconcileComplete").getStatus(); + LOGGER.debug("DataScienceCluster {} ReconcileComplete status: {}", resource.getMetadata().getName(), reconcileStatus); + dscReady = dscReady && reconcileStatus.equals("True"); + + // Wait for DataScienceClusterCreationSuccessful event + EventingAPIGroupDSL eventsClient = ResourceManager.getKubeClient().getClient().events(); + List resourceEvents = eventsClient.v1().events().inAnyNamespace().withNewFilter() + .withField("regarding.name", resource.getMetadata().getName()) + .withField("regarding.uid", resource.getMetadata().getUid()) + .endFilter().list().getItems(); + LOGGER.debug("DataScienceCluster {} events: {}", resource.getMetadata().getName(), resourceEvents.stream().map(Event::getReason).toList()); + boolean hasCreationSuccessfulEvent = resourceEvents.stream() + .anyMatch(resourceEvent -> Objects.equals(resourceEvent.getReason(), OdhConstants.DSC_CREATION_SUCCESSFUL_EVENT_NAME)); + dscReady = dscReady && hasCreationSuccessfulEvent; + return dscReady; }, () -> { });