Skip to content

Commit

Permalink
[ST] Refactor logs collecting of systemtests and collect more data (#…
Browse files Browse the repository at this point in the history 
…9637)

Signed-off-by: David Kornel <[email protected]>
  • Loading branch information
@kornys
kornys authored Feb 5, 2024
1 parent a86e99e commit 49c7deb
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 67 deletions.
117 changes: 66 additions & 51 deletions systemtest/src/main/java/io/strimzi/systemtest/logs/LogCollector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.extension.ExtensionContext;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
Expand Down Expand Up @@ -70,23 +71,22 @@ public class LogCollector {
}

private final KubeClient kubeClient;
private final File testSuite;
private final File testCase;
private final File logDir;
private final Path testSuitePath;
private final Path testCasePath;
private final String clusterOperatorNamespace;
private File namespaceFile;
private CollectorElement collectorElement;
private ExtensionContext extensionContext;
private Path namespacePath;
private final CollectorElement collectorElement;
private final ExtensionContext extensionContext;

public LogCollector(ExtensionContext extensionContext, CollectorElement collectorElement, KubeClient kubeClient, String logDir) throws IOException {
this.extensionContext = extensionContext;
this.collectorElement = collectorElement;
this.kubeClient = kubeClient;

this.logDir = new File(logDir + "/" + CURRENT_DATE);
final String logSuiteDir = this.logDir + "/" + collectorElement.getTestClassName();
Path logDirPath = Paths.get(logDir).resolve(CURRENT_DATE);
final Path logSuiteDir = logDirPath.resolve(collectorElement.getTestClassName());

this.testSuite = new File(logSuiteDir);
this.testSuitePath = logSuiteDir;

// contract only one Cluster Operator deployment inside all namespaces
Pod clusterOperatorPod = kubeClient.getClient().pods().inAnyNamespace().list().getItems().stream()
Expand All @@ -99,11 +99,11 @@ public LogCollector(ExtensionContext extensionContext, CollectorElement collecto
clusterOperatorPod.getMetadata().getNamespace() :
kubeClient.getNamespace();

this.testCase = new File(logSuiteDir + "/" + collectorElement.getTestMethodName());
this.testCasePath = logSuiteDir.resolve(collectorElement.getTestMethodName());

boolean logDirExist = this.logDir.exists() || this.logDir.mkdirs();
boolean logTestSuiteDirExist = this.testSuite.exists() || this.testSuite.mkdirs();
boolean logTestCaseDirExist = this.testCase.exists() || this.testCase.mkdirs();
boolean logDirExist = logDirPath.toFile().exists() || logDirPath.toFile().mkdirs();
boolean logTestSuiteDirExist = this.testSuitePath.toFile().exists() || this.testSuitePath.toFile().mkdirs();
boolean logTestCaseDirExist = this.testCasePath.toFile().exists() || this.testCasePath.toFile().mkdirs();

if (!logDirExist) {
throw new IOException("Unable to create path");
Expand All @@ -118,13 +118,13 @@ public LogCollector(ExtensionContext extensionContext, CollectorElement collecto

/**
* Core method, which collects logs from {@link #collectEvents(String)}, {@link #collectConfigMaps(String)},
* {@link #collectLogsFromPods(String)}, {@link #collectAllResourcesFromNamespace(String)} (String)},
* {@link #collectStrimzi(String)} and lastly {@link #collectClusterInfo(String)}.
* {@link #collectLogsFromPods(String)}, {@link #collectAllResourcesFromNamespace(String)},
* {@link #collectStrimzi(String)} and lastly {@link #collectClusterInfo()}.
*
* If anything fails in @BeforeAll or @AfterAll suite we gather these logs from @cde{this.testSuite}/@code{namespace}.
* Otherwise we collect it in code@{this.testCase}/code{namespace}.
* Otherwise, we collect it in code@{this.testCase}/code{namespace}.
*
* There are a few scenarios, which has to be take into account. Un-excepted exception was throw in:
* There are a few scenarios, which has to be taken into account. Un-excepted exception was throw in:
* 1.@BeforeAll scope
* 2.@BeforeEach scope
* 3.@Test scope
Expand All @@ -145,9 +145,9 @@ public synchronized void collect() {
namespaces = namespaces == null ? new HashSet<>() : namespaces;
namespaces.add(clusterOperatorNamespace);

// it's not test suite but test case, and we are gonna collect logs
// it's not test suite but test case, and we are going to collect logs
if (!this.collectorElement.getTestMethodName().isEmpty()) {
// @ParallelSuite -> this is generated namespace but we collect logs only iff STRIMZI_RBAC_SCOPE=CLUSTER
// @ParallelSuite -> this is generated namespace, but we collect logs only iff STRIMZI_RBAC_SCOPE=CLUSTER
// because when we run STRIMZI_RBAC_SCOPE=NAMESPACE mode we use one namespace (i.e., clusterOperatorNamespace).
if (!Environment.isNamespaceRbacScope()) {
// @IsolatedTest or @ParallelTest or @ParallelNamespaceTest -> are executed in that generated namespace
Expand Down Expand Up @@ -176,50 +176,54 @@ public synchronized void collect() {
// collect logs for all namespace related to test suite
namespaces.forEach(namespace -> {
if (this.collectorElement.getTestMethodName().isEmpty()) {
namespaceFile = new File(this.testSuite + "/" + namespace);
namespacePath = this.testSuitePath.resolve(namespace);
} else {
namespaceFile = new File(this.testCase + "/" + namespace);
namespacePath = this.testCasePath.resolve(namespace);
}

boolean namespaceLogDirExist = this.namespaceFile.exists() || this.namespaceFile.mkdirs();
boolean namespaceLogDirExist = this.namespacePath.toFile().exists() || this.namespacePath.toFile().mkdirs();
if (!namespaceLogDirExist) {
throw new RuntimeException("Unable to create path");
}

this.collectEvents(namespace);
this.collectConfigMaps(namespace);
this.collectSecrets(namespace);
this.collectDeployments(namespace);
this.collectLogsFromPods(namespace);
this.collectAllResourcesFromNamespace(namespace);
this.collectStrimzi(namespace);
this.collectClusterInfo(namespace);
this.collectOlm(namespace);
});

// collect cluster-wide information
this.collectClusterInfo();
}

private final void collectLogsForTestSuite(final Pod pod) {
private void collectLogsForTestSuite(final Pod pod) {
if (pod.getMetadata().getLabels().containsKey(TestConstants.TEST_SUITE_NAME_LABEL)) {
if (pod.getMetadata().getLabels().get(TestConstants.TEST_SUITE_NAME_LABEL).equals(StUtils.removePackageName(this.collectorElement.getTestClassName()))) {
LOGGER.debug("Collecting logs for TestSuite: {}, and Pod: {}/{}", this.collectorElement.getTestClassName(), pod.getMetadata().getNamespace(), pod.getMetadata().getName());
pod.getStatus().getContainerStatuses().forEach(
containerStatus -> scrapeAndCreateLogs(namespaceFile, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
containerStatus -> scrapeAndCreateLogs(namespacePath, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
}
// Tracing Pods (they can't be labeled because CR of the Jaeger does not propagate labels to the Pods )
} else if (pod.getMetadata().getName().contains("jaeger") || pod.getMetadata().getName().contains("cert-manager")) {
LOGGER.debug("Collecting logs for TestSuite: {}, and Jaeger Pods: {}/{}", this.collectorElement.getTestClassName(), pod.getMetadata().getNamespace(), pod.getMetadata().getName());
pod.getStatus().getContainerStatuses().forEach(
containerStatus -> scrapeAndCreateLogs(namespaceFile, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
containerStatus -> scrapeAndCreateLogs(namespacePath, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
}
}

private final void collectLogsForTestCase(final Pod pod) {
private void collectLogsForTestCase(final Pod pod) {
if (pod.getMetadata().getLabels().containsKey(TestConstants.TEST_CASE_NAME_LABEL)) {
// collect these Pods, which are deployed in that test case
// startWith is used because when we put inside Pod label with test case sometimes this test case exceed 63
// characters and we have to cut it to avoid exception
// characters, and we have to cut it to avoid exception
if (this.collectorElement.getTestMethodName().startsWith(pod.getMetadata().getLabels().get(TestConstants.TEST_CASE_NAME_LABEL))) {
LOGGER.debug("Collecting logs for TestCase: {}, and Pod: {}/{}", this.collectorElement.getTestMethodName(), pod.getMetadata().getNamespace(), pod.getMetadata().getName());
pod.getStatus().getContainerStatuses().forEach(
containerStatus -> scrapeAndCreateLogs(namespaceFile, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
containerStatus -> scrapeAndCreateLogs(namespacePath, pod.getMetadata().getName(), containerStatus, pod.getMetadata().getNamespace()));
}
}
}
Expand All @@ -233,7 +237,7 @@ private void collectLogsFromPods(String namespace) {
final String podName = pod.getMetadata().getName();
try {
pod.getStatus().getContainerStatuses().forEach(
containerStatus -> scrapeAndCreateLogs(namespaceFile, podName, containerStatus, namespace));
containerStatus -> scrapeAndCreateLogs(namespacePath, podName, containerStatus, namespace));
} catch (RuntimeException ex) {
LOGGER.warn("Failed to collect logs from Pod: {}/{}", namespace, podName);
}
Expand Down Expand Up @@ -263,14 +267,19 @@ private void collectEvents(String namespace) {
LOGGER.info("Collecting events in Namespace: {}", namespace);
String events = cmdKubeClient(namespace).getEvents();
// Write events to file
writeFile(namespaceFile + "/events.log", events);
writeFile(namespacePath.resolve("events.log"), events);
}

private void collectConfigMaps(String namespace) {
LOGGER.info("Collecting ConfigMaps in Namespace: {}", namespace);
kubeClient.listConfigMaps(namespace).forEach(configMap -> {
writeFile(namespaceFile + "/" + configMap.getMetadata().getName() + ".log", configMap.toString());
});
kubeClient.listConfigMaps(namespace).forEach(configMap ->
writeFile(namespacePath.resolve(configMap.getMetadata().getName() + "-configmap.log"), configMap.toString()));
}

private void collectSecrets(String namespace) {
LOGGER.info("Collecting Secrets in Namespace: {}", namespace);
kubeClient.listSecrets(namespace).forEach(secret ->
writeFile(namespacePath.resolve(secret.getMetadata().getName() + "-secret.log"), secret.toString()));
}

private void collectAllResourcesFromNamespace(String namespace) {
Expand All @@ -294,58 +303,64 @@ private void collectOlm(String namespace) {

private void collectResource(String kind, String namespace) {
LOGGER.info("Collecting: {} in Namespace: {}", kind, namespace);
writeFile(String.format("%s/%ss.log", namespaceFile, kind.toLowerCase(Locale.ROOT)), cmdKubeClient(namespace).getResourcesAsYaml(kind.toLowerCase(Locale.ROOT)));
writeFile(namespacePath.resolve(String.format("%ss.log", kind.toLowerCase(Locale.ROOT))), cmdKubeClient(namespace).getResourcesAsYaml(kind.toLowerCase(Locale.ROOT)));
}

private void collectStrimzi(String namespace) {
LOGGER.info("Collecting Strimzi resources in Namespace: {}", namespace);
String crData = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "strimzi", "-o", "yaml", "-n", namespaceFile.getName()).out();
writeFile(namespaceFile + "/strimzi-custom-resources.log", crData);
String crData = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "strimzi", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("strimzi-custom-resources.log"), crData);
}

private void collectDeployments(String namespace) {
LOGGER.info("Collecting Deployments resources in Namespace: {}", namespace);
String crData = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "deployment", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("deployments.log"), crData);
}

private void collectClusterInfo(String namespace) {
private void collectClusterInfo() {
LOGGER.info("Collecting cluster status");
String nodes = cmdKubeClient(namespace).exec(false, Level.DEBUG, "describe", "nodes").out();
writeFile(this.testSuite + "/cluster-status.log", nodes);
String nodes = cmdKubeClient().exec(false, Level.DEBUG, "describe", "nodes").out();
writeFile(this.testSuitePath.resolve("cluster-status.log"), nodes);
}

private void collectOperatorGroups(String namespace) {
LOGGER.info("Collecting OperatorGroups in Namespace: {}", namespace);
String operatorGroups = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "operatorGroups", "-o", "yaml", "-n", namespaceFile.getName()).out();
writeFile(namespaceFile + "/operator-groups.log", operatorGroups);
String operatorGroups = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "operatorGroups", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("operator-groups.log"), operatorGroups);
}

private void collectSubscriptions(String namespace) {
LOGGER.info("Collecting Subscriptions in Namespace: {}", namespace);
String subscriptions = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "subscriptions", "-o", "yaml", "-n", namespaceFile.getName()).out();
writeFile(namespaceFile + "/subscriptions.log", subscriptions);
String subscriptions = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "subscriptions", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("subscriptions.log"), subscriptions);
}

private void collectClusterServiceVersions(String namespace) {
LOGGER.info("Collecting ClusterServiceVersions in Namespace: {}", namespace);
String clusterServiceVersions = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "clusterServiceVersions", "-o", "yaml", "-n", namespaceFile.getName()).out();
writeFile(namespaceFile + "/cluster-service-versions.log", clusterServiceVersions);
String clusterServiceVersions = cmdKubeClient(namespace).exec(false, Level.DEBUG, "get", "clusterServiceVersions", "-o", "yaml", "-n", namespace).out();
writeFile(namespacePath.resolve("cluster-service-versions.log"), clusterServiceVersions);
}

private void scrapeAndCreateLogs(File path, String podName, ContainerStatus containerStatus, String namespace) {
private void scrapeAndCreateLogs(Path path, String podName, ContainerStatus containerStatus, String namespace) {
try {
String log = kubeClient.getPodResource(namespace, podName).inContainer(containerStatus.getName()).getLog();
// Write logs from containers to files
writeFile(path + "/logs-pod-" + podName + "-container-" + containerStatus.getName() + ".log", log);
writeFile(path.resolve("logs-pod-" + podName + "-container-" + containerStatus.getName() + ".log"), log);
} catch (RuntimeException e) {
LOGGER.warn("Unable to collect log from Pod: {}/{} and container: {} - Pod container is not initialized", namespace, podName, containerStatus.getName());
}

// Collect logs from previous version of the container
try {
String terminatedLog = kubeClient.getPodResource(namespace, podName).inContainer(containerStatus.getName()).terminated().getLog();
writeFile(path + "/logs-pod-" + podName + "-container-" + containerStatus.getName() + ".terminated.log", terminatedLog);
writeFile(path.resolve("logs-pod-" + podName + "-container-" + containerStatus.getName() + ".terminated.log"), terminatedLog);
} catch (RuntimeException e) {
// For most of the Pods it will fail as it doesn't have restarted container, so we just want to skip the exception
}

// Describe all Pods
String describe = cmdKubeClient(namespace).describe("pod", podName);
writeFile(path + "/describe-pod-" + podName + "-container-" + containerStatus.getName() + ".log", describe);
writeFile(path.resolve("describe-pod-" + podName + "-container-" + containerStatus.getName() + ".log"), describe);
}
}
28 changes: 12 additions & 16 deletions test/src/main/java/io/strimzi/test/TestUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,19 @@
import org.apache.logging.log4j.Logger;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.StringWriter;
import java.io.Writer;
import java.net.ServerSocket;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.AbstractMap;
Expand Down Expand Up @@ -365,24 +362,23 @@ public static <K, V> Map.Entry<K, V> entry(K key, V value) {
return new AbstractMap.SimpleEntry<>(key, value);
}

/** Method to create and write file */
public static void writeFile(Path filePath, String text) {
try {
Files.writeString(filePath, text, StandardCharsets.UTF_8);
} catch (IOException e) {
LOGGER.info("Exception during writing text in file");
e.printStackTrace();
}
}

/** Method to create and write file */
public static void writeFile(String filePath, String text) {
Writer writer = null;
try {
writer = new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(filePath), StandardCharsets.UTF_8));
writer.write(text);
Files.writeString(Paths.get(filePath), text, StandardCharsets.UTF_8);
} catch (IOException e) {
LOGGER.info("Exception during writing text in file");
e.printStackTrace();
} finally {
try {
if (writer != null) {
writer.close();
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
}

Expand Down

0 comments on commit 49c7deb

Please sign in to comment.