Skip to content

Commit

Permalink
[INLONG-8445][Sort] Support running tests on both Flink 1.13 and Flin…
Browse files Browse the repository at this point in the history 
…k 1.15 (#8475)
  • Loading branch information
@haibo-duan
haibo-duan authored Aug 1, 2023
1 parent 4d65939 commit a300d8d
Show file tree
Hide file tree
Showing 34 changed files with 1,973 additions and 174 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci_ut_flink15.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,12 @@ jobs:
restore-keys: ${{ runner.os }}-inlong-flink15

- name: Build for Flink 1.15 with Maven
run: mvn --update-snapshots -e -V package -U -pl inlong-sort/sort-core -am -Pv1.15 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000
run: mvn --update-snapshots -e -V clean install -U -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000
env:
CI: false

- name: Unit test for Flink 1.15 with Maven
run: mvn --update-snapshots -e -V clean test -am -pl inlong-sort/sort-core -Pv1.15
run: mvn --update-snapshots -e -V test -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15
env:
CI: false

Expand Down
4 changes: 1 addition & 3 deletions inlong-sort/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
<module>sort-core</module>
<module>sort-dist</module>
<module>sort-flink</module>
<module>sort-end-to-end-tests</module>
</modules>

<properties>
Expand Down Expand Up @@ -166,9 +167,6 @@
<activeByDefault>true</activeByDefault>
</activation>
<!-- End-To-End after refactoring, you need to mention this module under modules. -->
<modules>
<module>sort-end-to-end-tests</module>
</modules>
<properties>
<sort.flink.version>v1.13</sort.flink.version>
<flink.version>1.13.5</flink.version>
Expand Down
3 changes: 3 additions & 0 deletions inlong-sort/sort-dist/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,14 +145,17 @@
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-csv</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-json</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-sql-avro</artifactId>
<version>${flink.version}</version>
</dependency>
</dependencies>
</profile>
Expand Down
179 changes: 10 additions & 169 deletions inlong-sort/sort-end-to-end-tests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,187 +26,28 @@
</parent>

<artifactId>sort-end-to-end-tests</artifactId>
<packaging>pom</packaging>
<name>Apache InLong - Sort End to End Tests</name>

<properties>
<inlong.root.dir>${project.parent.parent.basedir}</inlong.root.dir>
</properties>

<dependencies>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mysql</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>kafka</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>clickhouse</artifactId>
<version>${testcontainers.version}</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-dist</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>${flink.test.utils.artifactId}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-flink-dependencies-${sort.flink.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>${flink.runtime.artifactId}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<profiles>
<profile>
<id>v1.13</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-dist</artifactId>
<version>${project.version}</version>
<destFileName>sort-dist.jar</destFileName>
<type>jar</type>
<outputDirectory>${project.build.directory}/dependencies</outputDirectory>
</artifactItem>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.jdbc.version}</version>
<destFileName>mysql-driver.jar</destFileName>
<type>jar</type>
<outputDirectory>${project.build.directory}/dependencies</outputDirectory>
</dependency>
<artifactItem>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-connector-kafka</artifactId>
<version>${project.version}</version>
<destFileName>sort-connector-kafka.jar</destFileName>
<type>jar</type>
<outputDirectory>${project.build.directory}/dependencies</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-connector-mysql-cdc</artifactId>
<version>${project.version}</version>
<destFileName>sort-connector-mysql-cdc.jar</destFileName>
<type>jar</type>
<outputDirectory>${project.build.directory}/dependencies</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.apache.inlong</groupId>
<artifactId>sort-connector-jdbc</artifactId>
<version>${project.version}</version>
<destFileName>sort-connector-jdbc.jar</destFileName>
<type>jar</type>
<outputDirectory>${project.build.directory}/dependencies</outputDirectory>
</artifactItem>
</artifactItems>
</configuration>
<executions>
<execution>
<id>copy-jars</id>
<goals>
<goal>copy</goal>
</goals>
<phase>pre-integration-test</phase>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>${plugin.failsafe.version}</version>
<executions>
<execution>
<id>end-to-end-tests</id>
<phase>integration-test</phase>
<configuration>
<includes>
<include>**/*.*</include>
</includes>
<forkCount>1</forkCount>
<systemPropertyVariables>
<moduleDir>${project.basedir}</moduleDir>
</systemPropertyVariables>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
<modules>
<module>sort-end-to-end-tests-v1.13</module>
</modules>
</profile>
<profile>
<id>v1.15</id>
<modules>
<module>sort-end-to-end-tests-v1.15</module>
</modules>
</profile>
</profiles>

Expand Down
Loading

0 comments on commit a300d8d

Please sign in to comment.