Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[ST] Refactor logs collecting of systemtests and collect more data #9637

Merged
merged 1 commit into from
Feb 5, 2024
Merged

[ST] Refactor logs collecting of systemtests and collect more data #9637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 66 additions & 51 deletions systemtest/src/main/java/io/strimzi/systemtest/logs/LogCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.extension.ExtensionContext;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
Expand Down Expand Up @@ -70,23 +71,22 @@ public class LogCollector {
}

private final KubeClient kubeClient;
private final File testSuite;
private final File testCase;
private final File logDir;
private final Path testSuitePath;
private final Path testCasePath;
private final String clusterOperatorNamespace;
private File namespaceFile;
private CollectorElement collectorElement;
private ExtensionContext extensionContext;
private Path namespacePath;
private final CollectorElement collectorElement;
private final ExtensionContext extensionContext;

public LogCollector(ExtensionContext extensionContext, CollectorElement collectorElement, KubeClient kubeClient, String logDir) throws IOException {
this.extensionContext = extensionContext;
this.collectorElement = collectorElement;
this.kubeClient = kubeClient;

this.logDir = new File(logDir + "/" + CURRENT_DATE);
final String logSuiteDir = this.logDir + "/" + collectorElement.getTestClassName();
Path logDirPath = Paths.get(logDir).resolve(CURRENT_DATE);
final Path logSuiteDir = logDirPath.resolve(collectorElement.getTestClassName());

this.testSuite = new File(logSuiteDir);
this.testSuitePath = logSuiteDir;

// contract only one Cluster Operator deployment inside all namespaces
Pod clusterOperatorPod = kubeClient.getClient().pods().inAnyNamespace().list().getItems().stream()
Expand All @@ -99,11 +99,11 @@ public LogCollector(ExtensionContext extensionContext, CollectorElement collecto
clusterOperatorPod.getMetadata().getNamespace() :
kubeClient.getNamespace();

this.testCase = new File(logSuiteDir + "/" + collectorElement.getTestMethodName());
this.testCasePath = logSuiteDir.resolve(collectorElement.getTestMethodName());

boolean logDirExist = this.logDir.exists() || this.logDir.mkdirs();
boolean logTestSuiteDirExist = this.testSuite.exists() || this.testSuite.mkdirs();
boolean logTestCaseDirExist = this.testCase.exists() || this.testCase.mkdirs();
boolean logDirExist = logDirPath.toFile().exists() || logDirPath.toFile().mkdirs();
boolean logTestSuiteDirExist = this.testSuitePath.toFile().exists() || this.testSuitePath.toFile().mkdirs();
boolean logTestCaseDirExist = this.testCasePath.toFile().exists() || this.testCasePath.toFile().mkdirs();

if (!logDirExist) {
throw new IOException("Unable to create path");
Expand All @@ -118,13 +118,13 @@ public LogCollector(ExtensionContext extensionContext, CollectorElement collecto

/**
* Core method, which collects logs from {@link #collectEvents(String)}, {@link #collectConfigMaps(String)},
* {@link #collectLogsFromPods(String)}, {@link #collectAllResourcesFromNamespace(String)} (String)},
* {@link #collectStrimzi(String)} and lastly {@link #collectClusterInfo(String)}.
* {@link #collectLogsFromPods(String)}, {@link #collectAllResourcesFromNamespace(String)},
* {@link #collectStrimzi(String)} and lastly {@link #collectClusterInfo()}.
*
* If anything fails in @BeforeAll or @AfterAll suite we gather these logs from @cde{this.testSuite}/@code{namespace}.
* Otherwise we collect it in code@{this.testCase}/code{namespace}.
* Otherwise, we collect it in code@{this.testCase}/code{namespace}.
*
* There are a few scenarios, which has to be take into account. Un-excepted exception was throw in:
* There are a few scenarios, which has to be taken into account. Un-excepted exception was throw in:
* 1.@BeforeAll scope
* 2.@BeforeEach scope
* 3.@Test scope
Expand All @@ -145,9 +145,9 @@ public synchronized void collect() {
namespaces = namespaces == null ? new HashSet<>() : namespaces;
namespaces.add(clusterOperatorNamespace);

// it's not test suite but test case, and we are gonna collect logs
// it's not test suite but test case, and we are going to collect logs
if (!this.collectorElement.getTestMethodName().isEmpty()) {
// @ParallelSuite -> this is generated namespace but we collect logs only iff STRIMZI_RBAC_SCOPE=CLUSTER
// @ParallelSuite -> this is generated namespace, but we collect logs only iff STRIMZI_RBAC_SCOPE=CLUSTER
// because when we run STRIMZI_RBAC_SCOPE=NAMESPACE mode we use one namespace (i.e., clusterOperatorNamespace).
if (!Environment.isNamespaceRbacScope()) {
// @IsolatedTest or @ParallelTest or @ParallelNamespaceTest -> are executed in that generated namespace
Expand Down Expand Up @@ -176,50 +176,54 @@ public synchronized void collect() {
// collect logs for all namespace related to test suite
namespaces.forEach(namespace -> {
if (this.collectorElement.getTestMethodName().isEmpty()) {
namespaceFile = new File(this.testSuite + "/" + namespace);
namespacePath = this.testSuitePath.resolve(namespace);
} else {
namespaceFile = new File(this.testCase + "/" + namespace);
namespacePath = this.testCasePath.resolve(namespace);
}

boolean namespaceLogDirExist = this.namespaceFile.exists() || this.namespaceFile.mkdirs();
boolean namespaceLogDirExist = this.namespacePath.toFile().exists() || this.namespacePath.toFile().mkdirs();
if (!namespaceLogDirExist) {
throw new RuntimeException("Unable to create path");
}

this.collectEvents(namespace);
this.collectConfigMaps(namespace);
this.collectSecrets(namespace);
this.collectDeployments(namespace);
this.collectLogsFromPods(namespace);
this.collectAllResourcesFromNamespace(namespace);
this.collectStrimzi(namespace);
this.collectClusterInfo(namespace);
this.collectOlm(namespace);
});

// collect cluster-wide information
this.collectClusterInfo();
}

private final void collectLogsForTestSuite(final Pod pod) {
private void collectLogsForTestSuite(final Pod pod) {
if (pod.getMetadata().getLabels().containsKey(TestConstants.TEST_SUITE_NAME_LABEL)) {
if (pod.getMetadata().getLabels().get(TestConstants.TEST_SUITE_NAME_LABEL).equals(StUtils.removePackageName(this.collectorElement.getTestClassName()))) {
LOGGER.debug("Collecting logs for TestSuite: {}, and Pod: {}/{}", this.collectorElement.getTestClassName(), pod.getMetadata().getNamespace(), pod.getMetadata().getName());
pod.getStatus().getContainerStatuses().forEach(
containerStatus -> scrapeAndCreateLogs(namespaceFile, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
containerStatus -> scrapeAndCreateLogs(namespacePath, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
}
// Tracing Pods (they can't be labeled because CR of the Jaeger does not propagate labels to the Pods )
} else if (pod.getMetadata().getName().contains("jaeger") || pod.getMetadata().getName().contains("cert-manager")) {
LOGGER.debug("Collecting logs for TestSuite: {}, and Jaeger Pods: {}/{}", this.collectorElement.getTestClassName(), pod.getMetadata().getNamespace(), pod.getMetadata().getName());
pod.getStatus().getContainerStatuses().forEach(
containerStatus -> scrapeAndCreateLogs(namespaceFile, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
containerStatus -> scrapeAndCreateLogs(namespacePath, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
}
}

private final void collectLogsForTestCase(final Pod pod) {
private void collectLogsForTestCase(final Pod pod) {
if (pod.getMetadata().getLabels().containsKey(TestConstants.TEST_CASE_NAME_LABEL)) {
// collect these Pods, which are deployed in that test case
// startWith is used because when we put inside Pod label with test case sometimes this test case exceed 63
// characters and we have to cut it to avoid exception
// characters, and we have to cut it to avoid exception
if (this.collectorElement.getTestMethodName().startsWith(pod.getMetadata().getLabels().get(TestConstants.TEST_CASE_NAME_LABEL))) {
LOGGER.debug("Collecting logs for TestCase: {}, and Pod: {}/{}", this.collectorElement.getTestMethodName(), pod.getMetadata().getNamespace(), pod.getMetadata().getName());
pod.getStatus().getContainerStatuses().forEach(
containerStatus -> scrapeAndCreateLogs(namespaceFile, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
containerStatus -> scrapeAndCreateLogs(namespacePath, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
}
}
}
Expand All @@ -233,7 +237,7 @@ private void collectLogsFromPods(String namespace) {
final String podName = pod.getMetadata().getName();
try {
pod.getStatus().getContainerStatuses().forEach(
containerStatus -> scrapeAndCreateLogs(namespaceFile, podName, containerStatus, namespace));
containerStatus -> scrapeAndCreateLogs(namespacePath, podName, containerStatus, namespace));
} catch (RuntimeException ex) {
LOGGER.warn("Failed to collect logs from Pod: {}/{}", namespace, podName);
}
Expand Down Expand Up @@ -263,14 +267,19 @@ private void collectEvents(String namespace) {
LOGGER.info("Collecting events in Namespace: {}", namespace);
String events = cmdKubeClient(namespace).getEvents();
// Write events to file
writeFile(namespaceFile + "/events.log", events);
writeFile(namespacePath.resolve("events.log"), events);
}

private void collectConfigMaps(String namespace) {
LOGGER.info("Collecting ConfigMaps in Namespace: {}", namespace);
kubeClient.listConfigMaps(namespace).forEach(configMap -> {
writeFile(namespaceFile + "/" + configMap.getMetadata().getName() + ".log", configMap.toString());
});
kubeClient.listConfigMaps(namespace).forEach(configMap ->
writeFile(namespacePath.resolve(configMap.getMetadata().getName() + "-configmap.log"), configMap.toString()));
}

private void collectSecrets(String namespace) {
LOGGER.info("Collecting Secrets in Namespace: {}", namespace);
kubeClient.listSecrets(namespace).forEach(secret ->
writeFile(namespacePath.resolve(secret.getMetadata().getName() + "-secret.log"), secret.toString()));
}

private void collectAllResourcesFromNamespace(String namespace) {
Expand All @@ -294,58 +303,64 @@ private void collectOlm(String namespace) {

private void collectResource(String kind, String namespace) {
LOGGER.info("Collecting: {} in Namespace: {}", kind, namespace);
writeFile(String.format("%s/%ss.log", namespaceFile, kind.toLowerCase(Locale.ROOT)), cmdKubeClient(namespace).getResourcesAsYaml(kind.toLowerCase(Locale.ROOT)));
writeFile(namespacePath.resolve(String.format("%ss.log", kind.toLowerCase(Locale.ROOT))), cmdKubeClient(namespace).getResourcesAsYaml(kind.toLowerCase(Locale.ROOT)));
}

private void collectStrimzi(String namespace) {
LOGGER.info("Collecting Strimzi resources in Namespace: {}", namespace);
String crData = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "strimzi", "-o", "yaml", "-n", namespaceFile.getName()).out();
writeFile(namespaceFile + "/strimzi-custom-resources.log", crData);
String crData = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "strimzi", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("strimzi-custom-resources.log"), crData);
}

private void collectDeployments(String namespace) {
LOGGER.info("Collecting Deployments resources in Namespace: {}", namespace);
String crData = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "deployment", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("deployments.log"), crData);
}

private void collectClusterInfo(String namespace) {
private void collectClusterInfo() {
LOGGER.info("Collecting cluster status");
String nodes = cmdKubeClient(namespace).exec(false, Level.DEBUG, "describe", "nodes").out();
writeFile(this.testSuite + "/cluster-status.log", nodes);
String nodes = cmdKubeClient().exec(false, Level.DEBUG, "describe", "nodes").out();
writeFile(this.testSuitePath.resolve("cluster-status.log"), nodes);
}

private void collectOperatorGroups(String namespace) {
LOGGER.info("Collecting OperatorGroups in Namespace: {}", namespace);
String operatorGroups = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "operatorGroups", "-o", "yaml", "-n", namespaceFile.getName()).out();
writeFile(namespaceFile + "/operator-groups.log", operatorGroups);
String operatorGroups = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "operatorGroups", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("operator-groups.log"), operatorGroups);
}

private void collectSubscriptions(String namespace) {
LOGGER.info("Collecting Subscriptions in Namespace: {}", namespace);
String subscriptions = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "subscriptions", "-o", "yaml", "-n", namespaceFile.getName()).out();
writeFile(namespaceFile + "/subscriptions.log", subscriptions);
String subscriptions = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "subscriptions", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("subscriptions.log"), subscriptions);
}

private void collectClusterServiceVersions(String namespace) {
LOGGER.info("Collecting ClusterServiceVersions in Namespace: {}", namespace);
String clusterServiceVersions = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "clusterServiceVersions", "-o", "yaml", "-n", namespaceFile.getName()).out();
writeFile(namespaceFile + "/cluster-service-versions.log", clusterServiceVersions);
String clusterServiceVersions = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "clusterServiceVersions", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("cluster-service-versions.log"), clusterServiceVersions);
}

private void scrapeAndCreateLogs(File path, String podName, ContainerStatus containerStatus, String namespace) {
private void scrapeAndCreateLogs(Path path, String podName, ContainerStatus containerStatus, String namespace) {
try {
String log = kubeClient.getPodResource(namespace, podName).inContainer(containerStatus.getName()).getLog();
// Write logs from containers to files
writeFile(path + "/logs-pod-" + podName + "-container-" + containerStatus.getName() + ".log", log);
writeFile(path.resolve("logs-pod-" + podName + "-container-" + containerStatus.getName() + ".log"), log);
} catch (RuntimeException e) {
LOGGER.warn("Unable to collect log from Pod: {}/{} and container: {} - Pod container is not initialized", namespace, podName, containerStatus.getName());
}

// Collect logs from previous version of the container
try {
String terminatedLog = kubeClient.getPodResource(namespace, podName).inContainer(containerStatus.getName()).terminated().getLog();
writeFile(path + "/logs-pod-" + podName + "-container-" + containerStatus.getName() + ".terminated.log", terminatedLog);
writeFile(path.resolve("logs-pod-" + podName + "-container-" + containerStatus.getName() + ".terminated.log"), terminatedLog);
} catch (RuntimeException e) {
// For most of the Pods it will fail as it doesn't have restarted container, so we just want to skip the exception
}

// Describe all Pods
String describe = cmdKubeClient(namespace).describe("pod", podName);
writeFile(path + "/describe-pod-" + podName + "-container-" + containerStatus.getName() + ".log", describe);
writeFile(path.resolve("describe-pod-" + podName + "-container-" + containerStatus.getName() + ".log"), describe);
}
}
28 changes: 12 additions & 16 deletions test/src/main/java/io/strimzi/test/TestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,19 @@
import org.apache.logging.log4j.Logger;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.StringWriter;
import java.io.Writer;
import java.net.ServerSocket;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.AbstractMap;
Expand Down Expand Up @@ -365,24 +362,23 @@ public static <K, V> Map.Entry<K, V> entry(K key, V value) {
return new AbstractMap.SimpleEntry<>(key, value);
}

/** Method to create and write file */
public static void writeFile(Path filePath, String text) {
try {
Files.writeString(filePath, text, StandardCharsets.UTF_8);
} catch (IOException e) {
LOGGER.info("Exception during writing text in file");
e.printStackTrace();
}
}

/** Method to create and write file */
public static void writeFile(String filePath, String text) {
Writer writer = null;
try {
writer = new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(filePath), StandardCharsets.UTF_8));
writer.write(text);
Files.writeString(Paths.get(filePath), text, StandardCharsets.UTF_8);
} catch (IOException e) {
LOGGER.info("Exception during writing text in file");
e.printStackTrace();
} finally {
try {
if (writer != null) {
writer.close();
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
}

Expand Down
Loading