diff --git a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java index 1c55b7b96d..1ac2502271 100644 --- a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java +++ b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java @@ -20,6 +20,7 @@ import com.ververica.cdc.connectors.tests.utils.JdbcProxy; import com.ververica.cdc.connectors.tests.utils.TestUtils; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -55,15 +56,17 @@ public class Db2E2eITCase extends FlinkContainerTestEnvironment { private static final Path db2CdcJar = TestUtils.getResource("db2-cdc-connector.jar"); private static final Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar"); + public static final String DB2_IMAGE = "ibmcom/db2"; + public static final String DB2_CUSTOM_IMAGE = "custom/db2-cdc:1.4"; private static final DockerImageName DEBEZIUM_DOCKER_IMAGE_NAME = DockerImageName.parse( - new ImageFromDockerfile("custom/db2-cdc:1.4") + new ImageFromDockerfile(DB2_CUSTOM_IMAGE) .withDockerfile(getFilePath("docker/db2/Dockerfile")) .get()) - .asCompatibleSubstituteFor("ibmcom/db2"); + .asCompatibleSubstituteFor(DB2_IMAGE); private static boolean db2AsnAgentRunning = false; - private Db2Container db2Container; + private static Db2Container db2Container; @Before public void before() { @@ -111,6 +114,23 @@ public void after() { super.after(); } + @AfterClass + public static void afterClass() { + // Cleanup the db2 image, because it's too large and will cause the next test to fail. + db2Container.getDockerClient().removeImageCmd(DB2_CUSTOM_IMAGE).exec(); + db2Container + .getDockerClient() + .listImagesCmd() + .withImageNameFilter(DB2_IMAGE) + .exec() + .forEach( + image -> + db2Container + .getDockerClient() + .removeImageCmd(image.getId()) + .exec()); + } + @Test public void testDb2CDC() throws Exception { List sqlLines = diff --git a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java index 11d185566c..e7a53396e7 100644 --- a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java +++ b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java @@ -20,6 +20,7 @@ import com.ververica.cdc.connectors.tests.utils.JdbcProxy; import com.ververica.cdc.connectors.tests.utils.TestUtils; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -53,13 +54,13 @@ public class OracleE2eITCase extends FlinkContainerTestEnvironment { private static final Path oracleCdcJar = TestUtils.getResource("oracle-cdc-connector.jar"); private static final Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar"); - - public OracleContainer oracle; + private static OracleContainer oracle; @Before public void before() { super.before(); LOG.info("Starting containers..."); + oracle = new OracleContainer(ORACLE_IMAGE) .withNetwork(NETWORK) @@ -77,6 +78,16 @@ public void after() { super.after(); } + @AfterClass + public static void afterClass() { + // Cleanup the oracle image, because it's too large and will cause the next test to fail. + oracle.getDockerClient() + .listImagesCmd() + .withImageNameFilter(ORACLE_IMAGE) + .exec() + .forEach(image -> oracle.getDockerClient().removeImageCmd(image.getId()).exec()); + } + @Test public void testOracleCDC() throws Exception { List sqlLines = diff --git a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java index 4dacfaa665..a31a3c7547 100644 --- a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java +++ b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java @@ -20,12 +20,14 @@ import com.ververica.cdc.connectors.tests.utils.JdbcProxy; import com.ververica.cdc.connectors.tests.utils.TestUtils; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.lifecycle.Startables; @@ -58,6 +60,7 @@ public class SqlServerE2eITCase extends FlinkContainerTestEnvironment { private static final Path sqlServerCdcJar = TestUtils.getResource("sqlserver-cdc-connector.jar"); private static final Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar"); + public static final String MSSQL_SERVER_IMAGE = "mcr.microsoft.com/mssql/server:2019-latest"; @Parameterized.Parameter(1) public boolean parallelismSnapshot; @@ -75,7 +78,7 @@ public static List parameters() { @Rule public MSSQLServerContainer sqlServer = - new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest") + new MSSQLServerContainer<>(MSSQL_SERVER_IMAGE) .withPassword("Password!") .withEnv("MSSQL_AGENT_ENABLED", "true") .withEnv("MSSQL_PID", "Standard") @@ -100,6 +103,12 @@ public void after() { super.after(); } + @AfterClass + public static void afterClass() { + // Cleanup the sqlserver image, because it's too large and will cause the next test to fail. + DockerClientFactory.instance().client().removeImageCmd(MSSQL_SERVER_IMAGE).exec(); + } + @Test public void testSqlServerCDC() throws Exception { List sqlLines = diff --git a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java index 5a117cc1b5..8a4756786c 100644 --- a/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java +++ b/flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java @@ -20,6 +20,7 @@ import com.ververica.cdc.connectors.tests.utils.JdbcProxy; import com.ververica.cdc.connectors.tests.utils.TestUtils; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; @@ -141,6 +142,16 @@ public void after() { super.after(); } + @AfterClass + public static void afterClass() { + // Cleanup the TIDB image, because it's too large and will cause the next test to fail. + TIDB.getDockerClient() + .listImagesCmd() + .withImageNameFilter("pingcap/tikv") + .exec() + .forEach(image -> TIDB.getDockerClient().removeImageCmd(image.getId()).exec()); + } + @Test public void testTIDBCDC() throws Exception { List sqlLines = diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml index eb23923a80..428c0c6692 100644 --- a/tools/azure-pipelines/jobs-template.yml +++ b/tools/azure-pipelines/jobs-template.yml @@ -80,14 +80,14 @@ jobs: module: mongodb sqlserver: module: sqlserver + tidb: + module: tidb db2: module: db2 vitess: module: vitess misc: module: misc - e2e_1: - module: e2e_1 steps: # download artifact from compile stage - task: DownloadPipelineArtifact@2 @@ -146,10 +146,8 @@ jobs: module: mysql oceanbase: module: oceanbase - tidb: - module: tidb - e2e_2: - module: e2e_2 + e2e: + module: e2e steps: # download artifact from compile stage - task: DownloadPipelineArtifact@2 diff --git a/tools/ci/stage.sh b/tools/ci/stage.sh index 98580d30f0..497b34d21e 100755 --- a/tools/ci/stage.sh +++ b/tools/ci/stage.sh @@ -23,8 +23,7 @@ STAGE_TIDB="tidb" STAGE_OCEANBASE="oceanbase" STAGE_DB2="db2" STAGE_VITESS="vitess" -STAGE_E2E_1="e2e_1" -STAGE_E2E_2="e2e_2" +STAGE_E2E="e2e" STAGE_MISC="misc" MODULES_MYSQL="\ @@ -97,12 +96,7 @@ function get_compile_modules_for_stage() { (${STAGE_VITESS}) echo "-pl $MODULES_VITESS -am" ;; - (${STAGE_E2E_1}) - # compile everything; using the -am switch does not work with negated module lists! - # the negation takes precedence, thus not all required modules would be built - echo "" - ;; - (${STAGE_E2E_2}) + (${STAGE_E2E}) # compile everything; using the -am switch does not work with negated module lists! # the negation takes precedence, thus not all required modules would be built echo "" @@ -136,7 +130,7 @@ function get_test_modules_for_stage() { local negated_tidb=\!${MODULES_TIDB//,/,\!} local negated_oceanbase=\!${MODULES_OCEANBASE//,/,\!} local negated_db2=\!${MODULES_DB2//,/,\!} - local negated_vitess=\!${MODULES_VITESS//,/,\!} + local negated_vitess=\!${MODULES_vitess//,/,\!} local negated_e2e=\!${MODULES_E2E//,/,\!} local modules_misc="$negated_mysql,$negated_postgres,$negated_oracle,$negated_mongodb,$negated_sqlserver,$negated_tidb,$negated_oceanbase,$negated_db2,$negated_vitess,$negated_e2e" @@ -168,11 +162,8 @@ function get_test_modules_for_stage() { (${STAGE_VITESS}) echo "-pl $modules_vitess" ;; - (${STAGE_E2E_1}) - echo "-pl $modules_e2e -Dtest=Db2E2eITCase,MongoE2eITCase,OracleE2eITCase,PostgresE2eITCase,SqlServerE2eITCas,VitessE2eITCase" - ;; - (${STAGE_E2E_2}) - echo "-pl $modules_e2e -Dtest=MySqlE2eITCase,TiDBE2eITCase" + (${STAGE_E2E}) + echo "-pl $modules_e2e" ;; (${STAGE_MISC}) echo "-pl $modules_misc"