Skip to content

Commit 85deb10

Browse files
gtk96gtk96
committed
refactor: create e2e-tests module, separate IT from unit tests
- New e2e-tests/ module with all *IT.java files (5 tests) - sync_database_mysql: added test-jar plugin to export test utilities - Updated root pom.xml with new module - CI: added e2e-tests to lint-and-unit matrix + dedicated e2e-tests job - Unit tests in original modules remain unaffected
1 parent d7c73c8 commit 85deb10

9 files changed

Lines changed: 194 additions & 1 deletion

File tree

‎.github/workflows/ci.yml‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
strategy:
4343
fail-fast: false
4444
matrix:
45-
module: [flink-demo, sync_database_mysql, flink-function, flink-paimon-demo]
45+
module: [flink-demo, sync_database_mysql, flink-function, flink-paimon-demo, e2e-tests]
4646
steps:
4747
- name: Checkout
4848
uses: actions/checkout@v7
@@ -176,6 +176,50 @@ jobs:
176176
if-no-files-found: ignore
177177
retention-days: 7
178178

179+
# ---------------------------------------------------------------------------
180+
# Job 2b: e2e tests (Testcontainers MySQL + MongoDB, requires Docker)
181+
# ---------------------------------------------------------------------------
182+
e2e-tests:
183+
name: E2E Tests
184+
runs-on: ubuntu-latest
185+
timeout-minutes: 30
186+
steps:
187+
- name: Checkout
188+
uses: actions/checkout@v7
189+
190+
- name: Set up JDK 11
191+
uses: actions/setup-java@v5
192+
with:
193+
java-version: '11'
194+
distribution: temurin
195+
196+
- name: Cache local Maven repository
197+
uses: actions/cache@v6
198+
with:
199+
path: ~/.m2/repository
200+
key: m2-${{ runner.os }}-jdk11-${{ hashFiles('**/pom.xml') }}
201+
restore-keys: |
202+
m2-${{ runner.os }}-jdk11-
203+
204+
- name: Build all modules (skip tests)
205+
run: |
206+
set +H
207+
./mvnw install -DskipTests -Dspotless.check.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true
208+
209+
- name: Run e2e tests
210+
run: |
211+
set +H
212+
./mvnw test -pl e2e-tests
213+
214+
- name: Upload surefire reports on failure
215+
if: failure()
216+
uses: actions/upload-artifact@v7
217+
with:
218+
name: surefire-e2e
219+
path: e2e-tests/target/surefire-reports
220+
if-no-files-found: ignore
221+
retention-days: 7
222+
179223
# ---------------------------------------------------------------------------
180224
# Job 3: package the runnable WDS jar (depends on lint-and-unit passing)
181225
# ---------------------------------------------------------------------------

‎e2e-tests/pom.xml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
<parent>
6+
<groupId>io.sophiadata</groupId>
7+
<artifactId>bigdata_code_tutorial</artifactId>
8+
<version>1.0.0</version>
9+
</parent>
10+
11+
<artifactId>e2e-tests</artifactId>
12+
<name>End-to-End Tests</name>
13+
<description>Integration and end-to-end tests requiring external services (MySQL, MongoDB, Docker)</description>
14+
15+
<dependencies>
16+
<!-- Source modules under test -->
17+
<dependency>
18+
<groupId>io.sophiadata</groupId>
19+
<artifactId>sync_database_mysql</artifactId>
20+
<version>${project.version}</version>
21+
<scope>test</scope>
22+
</dependency>
23+
<dependency>
24+
<groupId>io.sophiadata</groupId>
25+
<artifactId>sync_database_mysql</artifactId>
26+
<version>${project.version}</version>
27+
<type>test-jar</type>
28+
<scope>test</scope>
29+
</dependency>
30+
<dependency>
31+
<groupId>io.sophiadata</groupId>
32+
<artifactId>flink-paimon-demo</artifactId>
33+
<version>${project.version}</version>
34+
<scope>test</scope>
35+
</dependency>
36+
<dependency>
37+
<groupId>io.sophiadata</groupId>
38+
<artifactId>flink-demo</artifactId>
39+
<version>${project.version}</version>
40+
<scope>test</scope>
41+
</dependency>
42+
43+
<!-- Flink (provided) -->
44+
<dependency>
45+
<groupId>org.apache.flink</groupId>
46+
<artifactId>flink-streaming-java</artifactId>
47+
<version>${flink.version}</version>
48+
<scope>test</scope>
49+
</dependency>
50+
<dependency>
51+
<groupId>org.apache.flink</groupId>
52+
<artifactId>flink-table-api-java-bridge</artifactId>
53+
<version>${flink.version}</version>
54+
<scope>test</scope>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.apache.flink</groupId>
58+
<artifactId>flink-table-planner_2.12</artifactId>
59+
<version>${flink.version}</version>
60+
<scope>test</scope>
61+
</dependency>
62+
63+
<!-- Testcontainers -->
64+
<dependency>
65+
<groupId>org.testcontainers</groupId>
66+
<artifactId>junit-jupiter</artifactId>
67+
<version>${testcontainers.version}</version>
68+
<scope>test</scope>
69+
</dependency>
70+
<dependency>
71+
<groupId>org.testcontainers</groupId>
72+
<artifactId>mysql</artifactId>
73+
<version>${testcontainers.version}</version>
74+
<scope>test</scope>
75+
</dependency>
76+
<dependency>
77+
<groupId>org.testcontainers</groupId>
78+
<artifactId>mongodb</artifactId>
79+
<version>${testcontainers.version}</version>
80+
<scope>test</scope>
81+
</dependency>
82+
83+
<!-- JUnit -->
84+
<dependency>
85+
<groupId>org.junit.jupiter</groupId>
86+
<artifactId>junit-jupiter</artifactId>
87+
<scope>test</scope>
88+
</dependency>
89+
<dependency>
90+
<groupId>org.junit.vintage</groupId>
91+
<artifactId>junit-vintage-engine</artifactId>
92+
<scope>test</scope>
93+
</dependency>
94+
95+
<!-- H2 for CDBBatchSinkIT -->
96+
<dependency>
97+
<groupId>com.h2database</groupId>
98+
<artifactId>h2</artifactId>
99+
<version>2.2.224</version>
100+
<scope>test</scope>
101+
</dependency>
102+
103+
<!-- MongoDB BSON -->
104+
<dependency>
105+
<groupId>org.mongodb</groupId>
106+
<artifactId>mongodb-driver-sync</artifactId>
107+
<version>4.11.1</version>
108+
<scope>test</scope>
109+
</dependency>
110+
111+
<!-- MySQL driver -->
112+
<dependency>
113+
<groupId>com.mysql</groupId>
114+
<artifactId>mysql-connector-j</artifactId>
115+
<version>${mysql.version}</version>
116+
<scope>test</scope>
117+
</dependency>
118+
</dependencies>
119+
120+
<build>
121+
<plugins>
122+
<plugin>
123+
<groupId>org.apache.maven.plugins</groupId>
124+
<artifactId>maven-surefire-plugin</artifactId>
125+
<configuration>
126+
<!-- Run IT tests by default in this module -->
127+
<includes>
128+
<include>**/*IT.java</include>
129+
<include>**/*ITCase.java</include>
130+
</includes>
131+
<excludedGroups combine.self="override" />
132+
</configuration>
133+
</plugin>
134+
</plugins>
135+
</build>
136+
</project>

flink-paimon-demo/src/test/java/io/sophiadata/flink/paimon/mongo/MongoPaimonEndToEndIT.java renamed to e2e-tests/src/test/java/io/sophiadata/flink/paimon/mongo/MongoPaimonEndToEndIT.java

File renamed without changes.

flink-paimon-demo/src/test/java/io/sophiadata/flink/paimon/mongo/MongoToPaimonPipelineIT.java renamed to e2e-tests/src/test/java/io/sophiadata/flink/paimon/mongo/MongoToPaimonPipelineIT.java

File renamed without changes.

sync_database_mysql/src/test/java/io/sophiadata/flink/sync/CDBBatchSinkIT.java renamed to e2e-tests/src/test/java/io/sophiadata/flink/sync/CDBBatchSinkIT.java

File renamed without changes.

sync_database_mysql/src/test/java/io/sophiadata/flink/sync/SchemaEvolutionIT.java renamed to e2e-tests/src/test/java/io/sophiadata/flink/sync/SchemaEvolutionIT.java

File renamed without changes.

sync_database_mysql/src/test/java/io/sophiadata/flink/sync/sink/CreateMysqlLSinkTableIT.java renamed to e2e-tests/src/test/java/io/sophiadata/flink/sync/sink/CreateMysqlLSinkTableIT.java

File renamed without changes.

‎pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
<module>sync_database_mysql</module>
1616
<module>flink-function</module>
1717
<module>flink-paimon-demo</module>
18+
<module>e2e-tests</module>
1819
</modules>
1920

2021
<properties>

‎sync_database_mysql/pom.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,18 @@
410410
</execution>
411411
</executions>
412412
</plugin>
413+
<!-- Export test classes for e2e-tests module -->
414+
<plugin>
415+
<groupId>org.apache.maven.plugins</groupId>
416+
<artifactId>maven-jar-plugin</artifactId>
417+
<executions>
418+
<execution>
419+
<goals>
420+
<goal>test-jar</goal>
421+
</goals>
422+
</execution>
423+
</executions>
424+
</plugin>
413425
</plugins>
414426
</build>
415427

0 commit comments

Comments
 (0)