Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[feature] Add remove db docker image after test and revert e2e-2 #2508

Merged
merged 1 commit into from
Oct 10, 2023
Merged

[feature] Add remove db docker image after test and revert e2e-2 #2508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 23 additions & 3 deletions flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
import com.ververica.cdc.connectors.tests.utils.TestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
Expand Down Expand Up @@ -55,15 +56,17 @@ public class Db2E2eITCase extends FlinkContainerTestEnvironment {
private static final Path db2CdcJar = TestUtils.getResource("db2-cdc-connector.jar");
private static final Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar");

public static final String DB2_IMAGE = "ibmcom/db2";
public static final String DB2_CUSTOM_IMAGE = "custom/db2-cdc:1.4";
private static final DockerImageName DEBEZIUM_DOCKER_IMAGE_NAME =
DockerImageName.parse(
new ImageFromDockerfile("custom/db2-cdc:1.4")
new ImageFromDockerfile(DB2_CUSTOM_IMAGE)
.withDockerfile(getFilePath("docker/db2/Dockerfile"))
.get())
.asCompatibleSubstituteFor("ibmcom/db2");
.asCompatibleSubstituteFor(DB2_IMAGE);
private static boolean db2AsnAgentRunning = false;

private Db2Container db2Container;
private static Db2Container db2Container;

@Before
public void before() {
Expand Down Expand Up @@ -111,6 +114,23 @@ public void after() {
super.after();
}

@AfterClass
public static void afterClass() {
// Cleanup the db2 image, because it's too large and will cause the next test to fail.
db2Container.getDockerClient().removeImageCmd(DB2_CUSTOM_IMAGE).exec();
db2Container
.getDockerClient()
.listImagesCmd()
.withImageNameFilter(DB2_IMAGE)
.exec()
.forEach(
image ->
db2Container
.getDockerClient()
.removeImageCmd(image.getId())
.exec());
}

@Test
public void testDb2CDC() throws Exception {
List<String> sqlLines =
Expand Down
15 changes: 13 additions & 2 deletions flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
import com.ververica.cdc.connectors.tests.utils.TestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
Expand Down Expand Up @@ -53,13 +54,13 @@ public class OracleE2eITCase extends FlinkContainerTestEnvironment {

private static final Path oracleCdcJar = TestUtils.getResource("oracle-cdc-connector.jar");
private static final Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar");

public OracleContainer oracle;
private static OracleContainer oracle;

@Before
public void before() {
super.before();
LOG.info("Starting containers...");

oracle =
new OracleContainer(ORACLE_IMAGE)
.withNetwork(NETWORK)
Expand All @@ -77,6 +78,16 @@ public void after() {
super.after();
}

@AfterClass
public static void afterClass() {
// Cleanup the oracle image, because it's too large and will cause the next test to fail.
oracle.getDockerClient()
.listImagesCmd()
.withImageNameFilter(ORACLE_IMAGE)
.exec()
.forEach(image -> oracle.getDockerClient().removeImageCmd(image.getId()).exec());
}

@Test
public void testOracleCDC() throws Exception {
List<String> sqlLines =
Expand Down
11 changes: 10 additions & 1 deletion flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@
import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
import com.ververica.cdc.connectors.tests.utils.TestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.MSSQLServerContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.lifecycle.Startables;
Expand Down Expand Up @@ -58,6 +60,7 @@ public class SqlServerE2eITCase extends FlinkContainerTestEnvironment {
private static final Path sqlServerCdcJar =
TestUtils.getResource("sqlserver-cdc-connector.jar");
private static final Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar");
public static final String MSSQL_SERVER_IMAGE = "mcr.microsoft.com/mssql/server:2019-latest";

@Parameterized.Parameter(1)
public boolean parallelismSnapshot;
Expand All @@ -75,7 +78,7 @@ public static List<Object[]> parameters() {

@Rule
public MSSQLServerContainer sqlServer =
new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest")
new MSSQLServerContainer<>(MSSQL_SERVER_IMAGE)
.withPassword("Password!")
.withEnv("MSSQL_AGENT_ENABLED", "true")
.withEnv("MSSQL_PID", "Standard")
Expand All @@ -100,6 +103,12 @@ public void after() {
super.after();
}

@AfterClass
public static void afterClass() {
// Cleanup the sqlserver image, because it's too large and will cause the next test to fail.
DockerClientFactory.instance().client().removeImageCmd(MSSQL_SERVER_IMAGE).exec();
}

@Test
public void testSqlServerCDC() throws Exception {
List<String> sqlLines =
Expand Down
11 changes: 11 additions & 0 deletions flink-cdc-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
import com.ververica.cdc.connectors.tests.utils.TestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
Expand Down Expand Up @@ -141,6 +142,16 @@ public void after() {
super.after();
}

@AfterClass
public static void afterClass() {
// Cleanup the TIDB image, because it's too large and will cause the next test to fail.
TIDB.getDockerClient()
.listImagesCmd()
.withImageNameFilter("pingcap/tikv")
.exec()
.forEach(image -> TIDB.getDockerClient().removeImageCmd(image.getId()).exec());
}

@Test
public void testTIDBCDC() throws Exception {
List<String> sqlLines =
Expand Down
10 changes: 4 additions & 6 deletions tools/azure-pipelines/jobs-template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,14 +80,14 @@ jobs:
module: mongodb
sqlserver:
module: sqlserver
tidb:
module: tidb
db2:
module: db2
vitess:
module: vitess
misc:
module: misc
e2e_1:
module: e2e_1
steps:
# download artifact from compile stage
- task: DownloadPipelineArtifact@2
Expand Down Expand Up @@ -146,10 +146,8 @@ jobs:
module: mysql
oceanbase:
module: oceanbase
tidb:
module: tidb
e2e_2:
module: e2e_2
e2e:
module: e2e
steps:
# download artifact from compile stage
- task: DownloadPipelineArtifact@2
Expand Down
19 changes: 5 additions & 14 deletions tools/ci/stage.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@ STAGE_TIDB="tidb"
STAGE_OCEANBASE="oceanbase"
STAGE_DB2="db2"
STAGE_VITESS="vitess"
STAGE_E2E_1="e2e_1"
STAGE_E2E_2="e2e_2"
STAGE_E2E="e2e"
STAGE_MISC="misc"

MODULES_MYSQL="\
Expand Down Expand Up @@ -97,12 +96,7 @@ function get_compile_modules_for_stage() {
(${STAGE_VITESS})
echo "-pl $MODULES_VITESS -am"
;;
(${STAGE_E2E_1})
# compile everything; using the -am switch does not work with negated module lists!
# the negation takes precedence, thus not all required modules would be built
echo ""
;;
(${STAGE_E2E_2})
(${STAGE_E2E})
# compile everything; using the -am switch does not work with negated module lists!
# the negation takes precedence, thus not all required modules would be built
echo ""
Expand Down Expand Up @@ -136,7 +130,7 @@ function get_test_modules_for_stage() {
local negated_tidb=\!${MODULES_TIDB//,/,\!}
local negated_oceanbase=\!${MODULES_OCEANBASE//,/,\!}
local negated_db2=\!${MODULES_DB2//,/,\!}
local negated_vitess=\!${MODULES_VITESS//,/,\!}
local negated_vitess=\!${MODULES_vitess//,/,\!}
local negated_e2e=\!${MODULES_E2E//,/,\!}
local modules_misc="$negated_mysql,$negated_postgres,$negated_oracle,$negated_mongodb,$negated_sqlserver,$negated_tidb,$negated_oceanbase,$negated_db2,$negated_vitess,$negated_e2e"

Expand Down Expand Up @@ -168,11 +162,8 @@ function get_test_modules_for_stage() {
(${STAGE_VITESS})
echo "-pl $modules_vitess"
;;
(${STAGE_E2E_1})
echo "-pl $modules_e2e -Dtest=Db2E2eITCase,MongoE2eITCase,OracleE2eITCase,PostgresE2eITCase,SqlServerE2eITCas,VitessE2eITCase"
;;
(${STAGE_E2E_2})
echo "-pl $modules_e2e -Dtest=MySqlE2eITCase,TiDBE2eITCase"
(${STAGE_E2E})
echo "-pl $modules_e2e"
;;
(${STAGE_MISC})
echo "-pl $modules_misc"
Expand Down