[cdc][e2e] Unify logic to clean docker image when connector e2e test …

…finished (#2978) (cherry picked from commit a24c5d1)
apache · Jan 18, 2024 · 7ec53d3 · 7ec53d3
1 parent 6d7fb2f
commit 7ec53d3
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 68 deletions.
diff --git a/...-mysql-cdc/src/test/java/com/ververica/cdc/connectors/mysql/testutils/UniqueDatabase.java b/...-mysql-cdc/src/test/java/com/ververica/cdc/connectors/mysql/testutils/UniqueDatabase.java
@@ -142,7 +142,7 @@ public void createAndInitialize() {
         }
     }
 
-    /** Drop the database if is exists. */
+    /** Drop the database if it is existing. */
     public void dropDatabase() {
         try {
             try (Connection connection =

diff --git a/...k-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java b/...k-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java
@@ -20,7 +20,6 @@
 import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
 import com.ververica.cdc.connectors.tests.utils.TestUtils;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -114,23 +113,6 @@ public void after() {
         super.after();
     }
 
-    @AfterClass
-    public static void afterClass() {
-        // Cleanup the db2 image, because it's too large and will cause the next test to fail.
-        db2Container.getDockerClient().removeImageCmd(DB2_CUSTOM_IMAGE).exec();
-        db2Container
-                .getDockerClient()
-                .listImagesCmd()
-                .withImageNameFilter(DB2_IMAGE)
-                .exec()
-                .forEach(
-                        image ->
-                                db2Container
-                                        .getDockerClient()
-                                        .removeImageCmd(image.getId())
-                                        .exec());
-    }
-
     @Test
     public void testDb2CDC() throws Exception {
         List<String> sqlLines =

diff --git a/...source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OceanBaseE2eITCase.java b/...source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OceanBaseE2eITCase.java
@@ -16,17 +16,14 @@
 
 package com.ververica.cdc.connectors.tests;
 
-import com.github.dockerjava.api.DockerClient;
 import com.ververica.cdc.connectors.tests.utils.FlinkContainerTestEnvironment;
 import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
 import com.ververica.cdc.connectors.tests.utils.TestUtils;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.output.Slf4jLogConsumer;
 import org.testcontainers.containers.wait.strategy.Wait;
@@ -46,7 +43,6 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static org.junit.Assert.assertNotNull;
 
@@ -116,21 +112,6 @@ private Connection getTestConnection(String databaseName) {
         }
     }
 
-    @AfterClass
-    public static void afterClass() {
-        Stream.of(OB_SERVER, LOG_PROXY).forEach(GenericContainer::stop);
-
-        DockerClient client = DockerClientFactory.instance().client();
-        client.listImagesCmd()
-                .withImageNameFilter(OB_SERVER_IMAGE)
-                .exec()
-                .forEach(image -> client.removeImageCmd(image.getId()).exec());
-        client.listImagesCmd()
-                .withImageNameFilter(OB_LOG_PROXY_IMAGE)
-                .exec()
-                .forEach(image -> client.removeImageCmd(image.getId()).exec());
-    }
-
     @Test
     public void testOceanBaseCDC() throws Exception {
         List<String> sqlLines =

diff --git a/...dc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java b/...dc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java
@@ -20,7 +20,6 @@
 import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
 import com.ververica.cdc.connectors.tests.utils.TestUtils;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -84,16 +83,6 @@ public void after() {
         super.after();
     }
 
-    @AfterClass
-    public static void afterClass() {
-        // Cleanup the oracle image, because it's too large and will cause the next test to fail.
-        oracle.getDockerClient()
-                .listImagesCmd()
-                .withImageNameFilter(ORACLE_IMAGE)
-                .exec()
-                .forEach(image -> oracle.getDockerClient().removeImageCmd(image.getId()).exec());
-    }
-
     @Test
     public void testOracleCDC() throws Exception {
         List<String> sqlLines =

diff --git a/...source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java b/...source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java
@@ -20,14 +20,12 @@
 import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
 import com.ververica.cdc.connectors.tests.utils.TestUtils;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.MSSQLServerContainer;
 import org.testcontainers.containers.output.Slf4jLogConsumer;
 import org.testcontainers.lifecycle.Startables;
@@ -103,12 +101,6 @@ public void after() {
         super.after();
     }
 
-    @AfterClass
-    public static void afterClass() {
-        // Cleanup the sqlserver image, because it's too large and will cause the next test to fail.
-        DockerClientFactory.instance().client().removeImageCmd(MSSQL_SERVER_IMAGE).exec();
-    }
-
     @Test
     public void testSqlServerCDC() throws Exception {
         List<String> sqlLines =

diff --git a/...-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java b/...-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java
@@ -20,7 +20,6 @@
 import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
 import com.ververica.cdc.connectors.tests.utils.TestUtils;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Test;
@@ -142,16 +141,6 @@ public void after() {
         super.after();
     }
 
-    @AfterClass
-    public static void afterClass() {
-        // Cleanup the TIDB image, because it's too large and will cause the next test to fail.
-        TIDB.getDockerClient()
-                .listImagesCmd()
-                .withImageNameFilter("pingcap/tikv")
-                .exec()
-                .forEach(image -> TIDB.getDockerClient().removeImageCmd(image.getId()).exec());
-    }
-
     @Test
     public void testTIDBCDC() throws Exception {
         List<String> sqlLines =

diff --git a/...src/test/java/com/ververica/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java b/...src/test/java/com/ververica/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java
@@ -26,10 +26,12 @@
 import org.apache.flink.table.api.ValidationException;
 import org.apache.flink.util.TestLogger;
 
+import com.github.dockerjava.api.DockerClient;
 import com.ververica.cdc.connectors.mysql.testutils.MySqlContainer;
 import com.ververica.cdc.connectors.mysql.testutils.MySqlVersion;
 import com.ververica.cdc.connectors.mysql.testutils.UniqueDatabase;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Rule;
@@ -38,6 +40,7 @@
 import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.Container.ExecResult;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.Network;
@@ -174,6 +177,45 @@ public void after() {
         mysqlInventoryDatabase.dropDatabase();
     }
 
+    @AfterClass
+    public static void afterClass() {
+        DockerClient dockerClient = DockerClientFactory.instance().client();
+
+        // List all containers and remove the ones that are not testcontainers related.
+        dockerClient.listContainersCmd().exec().stream()
+                .filter(container -> !container.getImage().startsWith("testcontainers"))
+                .forEach(
+                        container -> {
+                            dockerClient.stopContainerCmd(container.getId()).exec();
+                            dockerClient.removeContainerCmd(container.getId()).exec();
+                        });
+
+        // List all images and remove the ones that are not flink、mysql、testcontainers related.
+        dockerClient.listImagesCmd().exec().stream()
+                .filter(
+                        image ->
+                                image.getRepoTags() != null
+                                        && Arrays.stream(image.getRepoTags())
+                                                .anyMatch(
+                                                        tag ->
+                                                                !tag.startsWith("flink:")
+                                                                        && !tag.startsWith(
+                                                                                "testcontainers")
+                                                                        && !tag.equals(
+                                                                                MYSQL
+                                                                                        .getDockerImageName())))
+                .forEach(
+                        image -> {
+                            try {
+                                dockerClient.removeImageCmd(image.getId()).exec();
+                            } catch (Exception e) {
+                                LOG.warn(
+                                        "Failed to remove image: {}",
+                                        String.join(",", image.getRepoTags()));
+                            }
+                        });
+    }
+
     /** Allow overriding the default flink properties. */
     public void overrideFlinkProperties(String properties) {
         jobManager.withEnv("FLINK_PROPERTIES", properties);