Skip to content

Commit 0f4d07d

Browse files
killme2008killme2008
authored
test: add integration tests and CI job (#99)
* test: add integration tests and CI job Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * ci: improve error and timeout Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: apply suggestions Signed-off-by: Dennis Zhuang <killme2008@gmail.com> --------- Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
1 parent 8253967 commit 0f4d07d

8 files changed

Lines changed: 872 additions & 0 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,51 @@ jobs:
7777
&& (mvn --projects ingester-protocol test
7878
|| mvn --projects ingester-protocol test
7979
|| mvn --projects ingester-protocol test)
80+
81+
integration_tests:
82+
needs: [test_ingester_common, test_ingester_grpc, test_ingester_protocol]
83+
runs-on: ubuntu-latest
84+
timeout-minutes: 10
85+
steps:
86+
- uses: actions/checkout@v4
87+
- name: Start GreptimeDB
88+
run: |
89+
docker run -d \
90+
-p 4000:4000 \
91+
-p 4001:4001 \
92+
-p 4002:4002 \
93+
-p 4003:4003 \
94+
--name greptimedb \
95+
greptime/greptimedb:v1.0.0-beta.4 standalone start \
96+
--http-addr 0.0.0.0:4000 \
97+
--rpc-bind-addr 0.0.0.0:4001 \
98+
--mysql-addr 0.0.0.0:4002 \
99+
--postgres-addr 0.0.0.0:4003
100+
- name: Wait for GreptimeDB
101+
run: |
102+
for i in {1..30}; do
103+
if curl -sf http://localhost:4000/health; then
104+
echo "GreptimeDB is ready"
105+
exit 0
106+
fi
107+
echo "Waiting for GreptimeDB... ($i/30)"
108+
sleep 2
109+
done
110+
echo "GreptimeDB failed to start"
111+
docker logs greptimedb
112+
exit 1
113+
- uses: actions/setup-java@v4
114+
with:
115+
java-version: '8'
116+
distribution: 'zulu'
117+
- name: Build
118+
run: mvn clean install -DskipTests -B -V
119+
- name: Integration Tests
120+
run: mvn verify -pl ingester-integration-tests -B
121+
env:
122+
GREPTIMEDB_ENDPOINTS: localhost:4001
123+
GREPTIMEDB_DATABASE: public
124+
GREPTIMEDB_JDBC_URL: jdbc:mysql://localhost:4002/public
125+
- name: Debug logs
126+
if: failure()
127+
run: docker logs greptimedb

‎ingester-integration-tests/pom.xml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
4+
Copyright 2023 Greptime Team
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
18+
-->
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>io.greptime</groupId>
23+
<artifactId>greptimedb-ingester</artifactId>
24+
<version>0.15.0</version>
25+
</parent>
26+
27+
<artifactId>ingester-integration-tests</artifactId>
28+
<name>${project.groupId}:${project.artifactId}</name>
29+
<description>Integration tests for GreptimeDB Java Ingester</description>
30+
31+
<dependencies>
32+
<dependency>
33+
<groupId>${project.groupId}</groupId>
34+
<artifactId>ingester-all</artifactId>
35+
<version>${project.version}</version>
36+
</dependency>
37+
38+
<!-- log impl -->
39+
<dependency>
40+
<groupId>org.apache.logging.log4j</groupId>
41+
<artifactId>log4j-api</artifactId>
42+
<scope>test</scope>
43+
</dependency>
44+
<dependency>
45+
<groupId>org.apache.logging.log4j</groupId>
46+
<artifactId>log4j-core</artifactId>
47+
<scope>test</scope>
48+
</dependency>
49+
<dependency>
50+
<groupId>org.apache.logging.log4j</groupId>
51+
<artifactId>log4j-slf4j-impl</artifactId>
52+
<scope>test</scope>
53+
</dependency>
54+
55+
<!-- MySQL JDBC driver for verification -->
56+
<dependency>
57+
<groupId>com.mysql</groupId>
58+
<artifactId>mysql-connector-j</artifactId>
59+
<version>8.4.0</version>
60+
<scope>test</scope>
61+
</dependency>
62+
63+
<!-- test -->
64+
<dependency>
65+
<groupId>junit</groupId>
66+
<artifactId>junit</artifactId>
67+
<scope>test</scope>
68+
</dependency>
69+
</dependencies>
70+
71+
<build>
72+
<plugins>
73+
<!-- Skip unit tests, only run integration tests -->
74+
<plugin>
75+
<groupId>org.apache.maven.plugins</groupId>
76+
<artifactId>maven-surefire-plugin</artifactId>
77+
<version>3.2.5</version>
78+
<configuration>
79+
<skip>true</skip>
80+
</configuration>
81+
</plugin>
82+
83+
<!-- Integration tests with failsafe plugin -->
84+
<plugin>
85+
<groupId>org.apache.maven.plugins</groupId>
86+
<artifactId>maven-failsafe-plugin</artifactId>
87+
<version>3.2.5</version>
88+
<executions>
89+
<execution>
90+
<goals>
91+
<goal>integration-test</goal>
92+
<goal>verify</goal>
93+
</goals>
94+
</execution>
95+
</executions>
96+
</plugin>
97+
</plugins>
98+
</build>
99+
</project>

‎ingester-integration-tests/src/test/java/io/greptime/BulkWriteIT.java‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
/*
2+
* Copyright 2023 Greptime Team
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.greptime;
18+
19+
import static org.junit.Assert.assertEquals;
20+
import io.greptime.models.DataType;
21+
import io.greptime.models.Table;
22+
import io.greptime.models.TableSchema;
23+
import java.sql.Connection;
24+
import org.junit.After;
25+
import org.junit.AfterClass;
26+
import org.junit.Before;
27+
import org.junit.BeforeClass;
28+
import org.junit.Test;
29+
import org.slf4j.Logger;
30+
import org.slf4j.LoggerFactory;
31+
32+
/**
33+
* Integration tests for Bulk Write API.
34+
*/
35+
public class BulkWriteIT {
36+
37+
private static final Logger LOG = LoggerFactory.getLogger(BulkWriteIT.class);
38+
private static final int ROW_COUNT = 100;
39+
40+
private static GreptimeDB client;
41+
private static Connection jdbcConn;
42+
43+
private String tableName;
44+
private TableSchema schema;
45+
46+
@BeforeClass
47+
public static void setupClass() throws Exception {
48+
client = ITHelper.createClient();
49+
jdbcConn = ITHelper.getJdbcConnection();
50+
LOG.info("Integration test client initialized");
51+
}
52+
53+
@AfterClass
54+
public static void teardownClass() {
55+
if (client != null) {
56+
client.shutdownGracefully();
57+
}
58+
if (jdbcConn != null) {
59+
try {
60+
jdbcConn.close();
61+
} catch (Exception e) {
62+
LOG.warn("Failed to close JDBC connection", e);
63+
}
64+
}
65+
}
66+
67+
@Before
68+
public void setup() throws Exception {
69+
tableName = ITHelper.uniqueTableName("test_bulk_write");
70+
schema = TableSchema.newBuilder(tableName)
71+
.addTag("host", DataType.String)
72+
.addTimestamp("ts", DataType.TimestampMillisecond)
73+
.addField("cpu_user", DataType.Float64)
74+
.addField("cpu_sys", DataType.Float64)
75+
.build();
76+
77+
// Bulk Write requires pre-existing table, create via DDL
78+
String createTableSql = String.format(
79+
"CREATE TABLE %s ("
80+
+ "host STRING,"
81+
+ "ts TIMESTAMP(3) TIME INDEX,"
82+
+ "cpu_user DOUBLE,"
83+
+ "cpu_sys DOUBLE,"
84+
+ "PRIMARY KEY (host)"
85+
+ ")",
86+
tableName);
87+
ITHelper.executeUpdate(jdbcConn, createTableSql);
88+
LOG.info("Created table: {}", tableName);
89+
}
90+
91+
@After
92+
public void teardown() {
93+
ITHelper.dropTableIfExists(jdbcConn, tableName);
94+
}
95+
96+
@Test
97+
public void testBulkWrite() throws Exception {
98+
BulkWrite.Config cfg = BulkWrite.Config.newBuilder()
99+
.allocatorInitReservation(0)
100+
.allocatorMaxAllocation(64 * 1024 * 1024L)
101+
.timeoutMsPerMessage(30000)
102+
.maxRequestsInFlight(4)
103+
.build();
104+
105+
try (BulkStreamWriter writer = client.bulkStreamWriter(schema, cfg)) {
106+
// Prepare test data with deterministic values
107+
Table.TableBufferRoot table = writer.tableBufferRoot(128);
108+
long baseTs = 1700000000000L;
109+
110+
for (int i = 0; i < ROW_COUNT; i++) {
111+
String host = "host_" + i;
112+
long ts = baseTs + i * 1000;
113+
double cpuUser = i * 0.1;
114+
double cpuSys = i * 0.05;
115+
table.addRow(host, ts, cpuUser, cpuSys);
116+
}
117+
table.complete();
118+
119+
// Write data
120+
Integer affectedRows = writer.writeNext().get();
121+
LOG.info("Bulk write affected rows: {}", affectedRows);
122+
123+
// Verify exact row count, not just > 0
124+
assertEquals("Should write exact row count", ROW_COUNT, affectedRows.intValue());
125+
126+
writer.completed();
127+
}
128+
129+
// Verify row count via JDBC
130+
int count = ITHelper.queryCount(jdbcConn, tableName);
131+
assertEquals("Row count should match", ROW_COUNT, count);
132+
133+
// Verify actual data content - spot check first, middle, and last rows
134+
ITHelper.verifyRow(jdbcConn, tableName, "host_0", 0.0, 0.0);
135+
ITHelper.verifyRow(jdbcConn, tableName, "host_50", 5.0, 2.5);
136+
ITHelper.verifyRow(jdbcConn, tableName, "host_99", 9.9, 4.95);
137+
138+
LOG.info("Verified {} rows with correct data in table {}", count, tableName);
139+
}
140+
141+
@Test
142+
public void testBulkWriteMultipleBatches() throws Exception {
143+
BulkWrite.Config cfg = BulkWrite.Config.newBuilder()
144+
.allocatorInitReservation(0)
145+
.allocatorMaxAllocation(64 * 1024 * 1024L)
146+
.timeoutMsPerMessage(30000)
147+
.maxRequestsInFlight(4)
148+
.build();
149+
150+
int batchCount = 5;
151+
int rowsPerBatch = 20;
152+
int totalWritten = 0;
153+
154+
try (BulkStreamWriter writer = client.bulkStreamWriter(schema, cfg)) {
155+
long baseTs = 1700000000000L;
156+
157+
for (int batch = 0; batch < batchCount; batch++) {
158+
Table.TableBufferRoot table = writer.tableBufferRoot(64);
159+
160+
for (int i = 0; i < rowsPerBatch; i++) {
161+
int rowNum = batch * rowsPerBatch + i;
162+
String host = "host_" + rowNum;
163+
long ts = baseTs + rowNum * 1000;
164+
double cpuUser = rowNum * 0.1;
165+
double cpuSys = rowNum * 0.05;
166+
table.addRow(host, ts, cpuUser, cpuSys);
167+
}
168+
table.complete();
169+
170+
Integer affectedRows = writer.writeNext().get();
171+
// Verify each batch writes exact count
172+
assertEquals("Batch " + batch + " should write exact count", rowsPerBatch, affectedRows.intValue());
173+
totalWritten += affectedRows;
174+
LOG.info("Batch {} wrote {} rows", batch, affectedRows);
175+
}
176+
177+
writer.completed();
178+
}
179+
180+
// Verify total written count
181+
int expectedCount = batchCount * rowsPerBatch;
182+
assertEquals("Total written should match", expectedCount, totalWritten);
183+
184+
// Verify row count via JDBC
185+
int count = ITHelper.queryCount(jdbcConn, tableName);
186+
assertEquals("Row count should match", expectedCount, count);
187+
188+
// Verify actual data content across batches
189+
ITHelper.verifyRow(jdbcConn, tableName, "host_0", 0.0, 0.0); // first row of batch 0
190+
ITHelper.verifyRow(jdbcConn, tableName, "host_40", 4.0, 2.0); // first row of batch 2
191+
ITHelper.verifyRow(jdbcConn, tableName, "host_99", 9.9, 4.95); // last row
192+
193+
LOG.info("Verified {} rows with correct data in table {}", count, tableName);
194+
}
195+
}

0 commit comments

Comments
 (0)