Skip to content

Commit

Permalink
Refactor odh-e2e to use test-frame and avoid duplication of code
Browse files Browse the repository at this point in the history 
Signed-off-by: David Kornel <[email protected]>
  • Loading branch information
@kornys
kornys committed May 6, 2024
1 parent 77ea3f7 commit dd609cf
Show file tree
Hide file tree
Showing 63 changed files with 594 additions and 2,999 deletions.
21 changes: 21 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
<aspectj.version>1.9.21.2</aspectj.version>
<allure.version>2.25.0</allure.version>
<allure.maven.version>2.12.0</allure.maven.version>
<test-frame.version>0.1.0-SNAPSHOT</test-frame.version>
</properties>

<repositories>
Expand All @@ -71,6 +72,11 @@
<name>GitHub Apache Maven Packages</name>
<url>https://maven.pkg.github.com/skodjob/opendatahub-crds</url>
</repository>
<repository>
<id>test-frame</id>
<name>GitHub Apache Maven Packages</name>
<url>https://maven.pkg.github.com/skodjob/test-frame</url>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
Expand All @@ -93,6 +99,21 @@
</dependencyManagement>

<dependencies>
<dependency>
<groupId>io.skodjob</groupId>
<artifactId>test-frame-common</artifactId>
<version>${test-frame.version}</version>
</dependency>
<dependency>
<groupId>io.skodjob</groupId>
<artifactId>test-frame-kubernetes</artifactId>
<version>${test-frame.version}</version>
</dependency>
<dependency>
<groupId>io.skodjob</groupId>
<artifactId>test-frame-openshift</artifactId>
<version>${test-frame.version}</version>
</dependency>
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>openshift-client</artifactId>
Expand Down
5 changes: 5 additions & 0 deletions settings.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@
<username>x-access-token</username>
<password>${env.GITHUB_TOKEN}</password>
</server>
<server>
<id>test-frame</id>
<username>x-access-token</username>
<password>${env.GITHUB_TOKEN}</password>
</server>
<server>
<id>test-metadata-generator</id>
<username>x-access-token</username>
Expand Down
5 changes: 0 additions & 5 deletions src/main/java/io/odh/test/Environment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,6 @@ public class Environment {
public static final String USER_PATH = System.getProperty("user.dir");

private static final String CONFIG_FILE_PATH_ENV = "ENV_FILE";
private static final String TOKEN_ENV = "KUBE_TOKEN";
private static final String URL_ENV = "KUBE_URL";
private static final String PRODUCT_ENV = "PRODUCT";
private static final String LOG_DIR_ENV = "LOG_DIR";

Expand Down Expand Up @@ -74,9 +72,6 @@ public class Environment {
* Set values
*/
public static final String PRODUCT = getOrDefault(PRODUCT_ENV, PRODUCT_ODH);
public static final String RUN_USER = getOrDefault("USER", null);
public static final String KUBE_TOKEN = getOrDefault(TOKEN_ENV, null);
public static final String KUBE_URL = getOrDefault(URL_ENV, null);

//Install
public static final boolean SKIP_INSTALL_OPERATOR_DEPS = getOrDefault(SKIP_INSTALL_OPERATOR_DEPS_ENV, Boolean::valueOf, false);
Expand Down
197 changes: 80 additions & 117 deletions src/main/java/io/odh/test/TestUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,32 +7,36 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.InvalidFormatException;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import io.odh.test.framework.WaitException;
import io.fabric8.kubernetes.api.model.EndpointSubset;
import io.fabric8.kubernetes.api.model.Endpoints;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.dsl.Resource;
import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlan;
import io.opendatahub.datasciencecluster.v1.datascienceclusterstatus.Conditions;
import io.skodjob.testframe.resources.KubeResourceManager;
import io.skodjob.testframe.utils.KubeUtils;
import io.skodjob.testframe.wait.Wait;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.BooleanSupplier;

import static io.odh.test.TestConstants.GLOBAL_POLL_INTERVAL_SHORT;
import static io.odh.test.TestConstants.GLOBAL_TIMEOUT;
Expand All @@ -56,79 +60,6 @@ private TestUtils() {
// All static methods
}

/**
* Poll the given {@code ready} function every {@code pollIntervalMs} milliseconds until it returns true,
* or throw a WaitException if it doesn't return true within {@code timeoutMs} milliseconds.
*
* @return The remaining time left until timeout occurs
* (helpful if you have several calls which need to share a common timeout),
*/
public static long waitFor(String description, long pollIntervalMs, long timeoutMs, BooleanSupplier ready) {
return waitFor(description, pollIntervalMs, timeoutMs, ready, () -> { });
}

public static long waitFor(String description, long pollIntervalMs, long timeoutMs, BooleanSupplier ready, Runnable onTimeout) {
LOGGER.debug("Waiting for {}", description);
long deadline = System.currentTimeMillis() + timeoutMs;

String exceptionMessage = null;
String previousExceptionMessage = null;

// in case we are polling every 1s, we want to print exception after x tries, not on the first try
// for minutes poll interval will 2 be enough
int exceptionAppearanceCount = Duration.ofMillis(pollIntervalMs).toMinutes() > 0 ? 2 : Math.max((int) (timeoutMs / pollIntervalMs) / 4, 2);
int exceptionCount = 0;
int newExceptionAppearance = 0;

StringWriter stackTraceError = new StringWriter();

while (true) {
boolean result;
try {
result = ready.getAsBoolean();
} catch (Exception e) {
exceptionMessage = e.getMessage();

if (++exceptionCount == exceptionAppearanceCount && exceptionMessage != null && exceptionMessage.equals(previousExceptionMessage)) {
LOGGER.error("While waiting for {} exception occurred: {}", description, exceptionMessage);
// log the stacktrace
e.printStackTrace(new PrintWriter(stackTraceError));
} else if (exceptionMessage != null && !exceptionMessage.equals(previousExceptionMessage) && ++newExceptionAppearance == 2) {
previousExceptionMessage = exceptionMessage;
}

result = false;
}
long timeLeft = deadline - System.currentTimeMillis();
if (result) {
return timeLeft;
}
if (timeLeft <= 0) {
if (exceptionCount > 1) {
LOGGER.error("Exception waiting for {}, {}", description, exceptionMessage);

if (!stackTraceError.toString().isEmpty()) {
// printing handled stacktrace
LOGGER.error(stackTraceError.toString());
}
}
onTimeout.run();
WaitException waitException = new WaitException("Timeout after " + timeoutMs + " ms waiting for " + description);
waitException.printStackTrace();
throw waitException;
}
long sleepTime = Math.min(pollIntervalMs, timeLeft);
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("{} not ready, will try again in {} ms ({}ms till timeout)", description, sleepTime, timeLeft);
}
try {
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
return deadline - System.currentTimeMillis();
}
}
}

/**
* Polls the given HTTP {@code url} until it gives != 503 status code
*/
Expand All @@ -141,7 +72,7 @@ public static void waitForServiceNotUnavailable(String url) {
}

public static void waitForServiceNotUnavailable(HttpClient httpClient, String url) {
TestUtils.waitFor("service to be not unavailable", GLOBAL_POLL_INTERVAL_SHORT, GLOBAL_TIMEOUT, () -> {
Wait.until("service to be not unavailable", GLOBAL_POLL_INTERVAL_SHORT, GLOBAL_TIMEOUT, () -> {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.GET()
Expand All @@ -167,42 +98,6 @@ public Thread newThread(Runnable r) {
}
});

public static CompletableFuture<Void> asyncWaitFor(String description, long pollIntervalMs, long timeoutMs, BooleanSupplier ready) {
LOGGER.info("Waiting for {}", description);
long deadline = System.currentTimeMillis() + timeoutMs;
CompletableFuture<Void> future = new CompletableFuture<>();
Executor delayed = CompletableFuture.delayedExecutor(pollIntervalMs, TimeUnit.MILLISECONDS, EXECUTOR);
Runnable r = new Runnable() {
@Override
public void run() {
boolean result;
try {
result = ready.getAsBoolean();
} catch (Exception e) {
future.completeExceptionally(e);
return;
}
long timeLeft = deadline - System.currentTimeMillis();
if (!future.isDone()) {
if (!result) {
if (timeLeft >= 0) {
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("{} not ready, will try again ({}ms till timeout)", description, timeLeft);
}
delayed.execute(this);
} else {
future.completeExceptionally(new TimeoutException(String.format("Waiting for %s timeout %s exceeded", description, timeoutMs)));
}
} else {
future.complete(null);
}
}
}
};
r.run();
return future;
}

public static InputStream getFileFromResourceAsStream(String fileName) {

// The class loader that loaded the class
Expand Down Expand Up @@ -272,4 +167,72 @@ public static <T> T runUntilPass(int retry, Callable<T> fn) {
}
throw new IllegalStateException(String.format("Command wasn't pass in %s attempts", retry));
}

public static io.opendatahub.datasciencecluster.v1.datascienceclusterstatus.Conditions getDscConditionByType(List<Conditions> conditions, String type) {
return conditions.stream().filter(c -> c.getType().equals(type)).findFirst().orElseGet(null);
}

public static org.kubeflow.v1.notebookstatus.Conditions getNotebookConditionByType(List<org.kubeflow.v1.notebookstatus.Conditions> conditions, String type) {
return conditions.stream().filter(c -> c.getType().equals(type)).findFirst().orElseGet(null);
}

public static io.kserve.serving.v1beta1.inferenceservicestatus.Conditions getInferenceServiceConditionByType(List<io.kserve.serving.v1beta1.inferenceservicestatus.Conditions> conditions, String type) {
return conditions.stream().filter(c -> c.getType().equals(type)).findFirst().orElseGet(null);
}

public static void clearOdhRemainingResources() {
KubeResourceManager.getKubeClient().getClient().apiextensions().v1().customResourceDefinitions().list().getItems()
.stream().filter(crd -> crd.getMetadata().getName().contains("opendatahub.io")).toList()
.forEach(crd -> {
LOGGER.info("Deleting CRD {}", crd.getMetadata().getName());
KubeResourceManager.getKubeClient().getClient().resource(crd).delete();
});
KubeResourceManager.getKubeClient().getClient().namespaces().withName("opendatahub").delete();
}

/**
* TODO - this should be removed when https://github.com/opendatahub-io/opendatahub-operator/issues/765 will be resolved
*/
public static void deleteDefaultDSCI() {
LOGGER.info("Clearing DSCI ...");
KubeResourceManager.getKubeCmdClient().exec(false, true, Long.valueOf(GLOBAL_TIMEOUT).intValue(), "delete", "dsci", "--all");
}

public static void waitForInstallPlan(String namespace, String csvName) {
Wait.until(String.format("Install plan with new version: %s:%s", namespace, csvName),
GLOBAL_POLL_INTERVAL_SHORT, GLOBAL_TIMEOUT, () -> {
try {
InstallPlan ip = KubeUtils.getNonApprovedInstallPlan(namespace, csvName);
LOGGER.debug("Found InstallPlan {} - {}", ip.getMetadata().getName(), ip.getSpec().getClusterServiceVersionNames());
return true;
} catch (NoSuchElementException ex) {
LOGGER.debug("No new install plan available. Checking again ...");
return false;
}
}, () -> { });
}

public static void waitForEndpoints(String name, Resource<Endpoints> endpoints) {
Wait.until("%s service endpoints to come up".formatted(name), GLOBAL_POLL_INTERVAL_SHORT, GLOBAL_TIMEOUT, () -> {
try {
Endpoints endpointset = endpoints.get();
if (endpointset == null) {
return false;
}
List<EndpointSubset> subsets = endpointset.getSubsets();
if (subsets.isEmpty()) {
return false;
}
for (EndpointSubset subset : subsets) {
return !subset.getAddresses().isEmpty();
}
} catch (KubernetesClientException e) {
if (e.getCode() == 404) {
return false;
}
throw e;
}
return false;
});
}
}
15 changes: 0 additions & 15 deletions src/main/java/io/odh/test/framework/WaitException.java
View file

This file was deleted.

4 changes: 2 additions & 2 deletions src/main/java/io/odh/test/framework/listeners/OdhResourceCleaner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
package io.odh.test.framework.listeners;

import io.odh.test.Environment;
import io.odh.test.platform.KubeUtils;
import io.odh.test.TestUtils;
import org.junit.jupiter.api.extension.AfterAllCallback;
import org.junit.jupiter.api.extension.ExtensionContext;

Expand All @@ -14,7 +14,7 @@ public class OdhResourceCleaner implements AfterAllCallback {
@Override
public void afterAll(ExtensionContext extensionContext) {
if (!Environment.SKIP_INSTALL_OPERATOR && !Environment.SKIP_DEPLOY_DSCI_DSC) {
KubeUtils.clearOdhRemainingResources();
TestUtils.clearOdhRemainingResources();
}
}
}
38 changes: 0 additions & 38 deletions src/main/java/io/odh/test/framework/listeners/ResourceManagerContextHandler.java
View file

This file was deleted.

Loading

0 comments on commit dd609cf

Please sign in to comment.