Skip to content

Commit a8303a5

Browse files
CaideyipiCaideyipi
committed
Merge branch 'master' of https://github.com/apache/iotdb into last-cache-fix
2 parents 47f7180 + 4ac0b7e commit a8303a5

1,444 files changed

Lines changed: 37759 additions & 11847 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/cluster-it-1c1d.yml‎

Lines changed: 65 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,52 @@ env:
3030
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
3131

3232
jobs:
33-
Simple:
33+
# Ubuntu runs all ITs in a single job (already fast at ~49 min)
34+
Ubuntu:
35+
runs-on: ubuntu-latest
36+
37+
steps:
38+
- uses: actions/checkout@v5
39+
- name: Set up JDK
40+
uses: actions/setup-java@v5
41+
with:
42+
distribution: corretto
43+
java-version: 17
44+
env:
45+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
46+
- name: Cache Maven packages
47+
uses: actions/cache@v5
48+
with:
49+
path: ~/.m2
50+
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
51+
restore-keys: ${{ runner.os }}-m2-
52+
- name: Adjust Linux kernel somaxconn
53+
shell: bash
54+
run: sudo sysctl -w net.core.somaxconn=65535
55+
- name: IT/UT Test
56+
shell: bash
57+
run: |
58+
mvn clean verify \
59+
-P with-integration-tests \
60+
-DskipUTs \
61+
-DintegrationTest.forkCount=2 \
62+
-pl integration-test \
63+
-am
64+
- name: Upload Artifact
65+
if: failure()
66+
uses: actions/upload-artifact@v6
67+
with:
68+
name: standalone-log-Linux
69+
path: integration-test/target/cluster-logs
70+
retention-days: 1
71+
72+
# Windows is ~77% slower than Ubuntu, so split into 3 shards to parallelize
73+
Windows:
3474
strategy:
3575
fail-fast: false
36-
max-parallel: 15
3776
matrix:
38-
os: [ubuntu-latest, windows-latest]
39-
runs-on: ${{ matrix.os }}
77+
shard: [0, 1, 2]
78+
runs-on: windows-latest
4079

4180
steps:
4281
- uses: actions/checkout@v5
@@ -54,36 +93,47 @@ jobs:
5493
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
5594
restore-keys: ${{ runner.os }}-m2-
5695
- name: Adjust network dynamic TCP ports range
57-
if: ${{ runner.os == 'Windows' }}
5896
shell: pwsh
5997
run: |
6098
netsh int ipv4 set dynamicport tcp start=32768 num=32768
6199
netsh int ipv4 set dynamicport udp start=32768 num=32768
62100
netsh int ipv6 set dynamicport tcp start=32768 num=32768
63101
netsh int ipv6 set dynamicport udp start=32768 num=32768
64-
- name: Adjust Linux kernel somaxconn
65-
if: ${{ runner.os == 'Linux' }}
102+
- name: Build IT shard list
66103
shell: bash
67-
run: sudo sysctl -w net.core.somaxconn=65535
68-
# - name: Adjust Mac kernel somaxconn
69-
# if: ${{ runner.os == 'macOS' }}
70-
# shell: bash
71-
# run: sudo sysctl -w kern.ipc.somaxconn=65535
104+
# Distribute LocalStandaloneIT test classes across 3 shards using hash-mod assignment.
105+
# The list is written to a file so failsafe.includesFile can read it without command-line length limits.
106+
run: |
107+
set -euo pipefail
108+
SHARD=${{ matrix.shard }}
109+
TOTAL=3
110+
# Write outside the repo so Apache RAT (license check) doesn't flag the file.
111+
# Using a single grep -rl call instead of `find | xargs grep`: on Windows Git Bash,
112+
# ARG_MAX is small so xargs batches the file list, and any batch with no matches
113+
# makes grep exit 1, which makes xargs exit 123 and trips `set -o pipefail`.
114+
grep -rlE --include='*IT.java' '\bLocalStandaloneIT\b' integration-test/src/test/java \
115+
| awk -F'/' '{print $NF}' | sed 's/\.java$//' \
116+
| sort \
117+
| awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
118+
> "$RUNNER_TEMP/it-shard.txt"
119+
echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes"
120+
head -5 "$RUNNER_TEMP/it-shard.txt"
72121
- name: IT/UT Test
73122
shell: bash
74-
# we do not compile client-cpp for saving time, it is tested in client.yml
75-
# we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
76123
run: |
77124
mvn clean verify \
78125
-P with-integration-tests \
79126
-DskipUTs \
80127
-DintegrationTest.forkCount=2 \
128+
-Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
129+
-DfailIfNoTests=false \
130+
-Dfailsafe.failIfNoSpecifiedTests=false \
81131
-pl integration-test \
82132
-am
83133
- name: Upload Artifact
84134
if: failure()
85135
uses: actions/upload-artifact@v6
86136
with:
87-
name: standalone-log-java${{ matrix.java }}-${{ runner.os }}
137+
name: standalone-log-Windows-shard${{ matrix.shard }}
88138
path: integration-test/target/cluster-logs
89139
retention-days: 1

‎.github/workflows/compile-check.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,7 @@ jobs:
5353
shell: bash
5454
run: |
5555
mvn clean package -P with-integration-tests -DskipTests -ntp
56+
- name: Compiler Test (Chinese i18n)
57+
shell: bash
58+
run: |
59+
mvn clean package -P with-integration-tests,with-zh-locale -DskipTests -ntp

‎.github/workflows/pipe-it.yml‎

Lines changed: 110 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ jobs:
119119
name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
120120
path: integration-test/target/cluster-logs
121121
retention-days: 30
122+
# 12 IT classes split across 3 parallel shards to cut the historical ~42 min
123+
# wall clock to ~14 min. See cluster-it-1c1d.yml for the shard pattern.
122124
dual-tree-auto-basic:
123125
strategy:
124126
fail-fast: false
@@ -128,6 +130,7 @@ jobs:
128130
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
129131
cluster: [HighPerformanceMode]
130132
os: [ubuntu-latest]
133+
shard: [0, 1, 2]
131134
runs-on: ${{ matrix.os }}
132135
steps:
133136
- uses: actions/checkout@v5
@@ -147,6 +150,21 @@ jobs:
147150
- name: Sleep for a random duration between 0 and 10000 milliseconds
148151
run: |
149152
sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
153+
- name: Build IT shard list
154+
shell: bash
155+
# See cluster-it-1c1d.yml for the shard-list pattern. Write under
156+
# $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
157+
run: |
158+
set -euo pipefail
159+
SHARD=${{ matrix.shard }}
160+
TOTAL=3
161+
grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTreeAutoBasic\b' integration-test/src/test/java \
162+
| awk -F'/' '{print $NF}' | sed 's/\.java$//' \
163+
| sort \
164+
| awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
165+
> "$RUNNER_TEMP/it-shard.txt"
166+
echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes"
167+
head -5 "$RUNNER_TEMP/it-shard.txt"
150168
- name: IT Test
151169
shell: bash
152170
# we do not compile client-cpp for saving time, it is tested in client.yml
@@ -164,6 +182,9 @@ jobs:
164182
-DskipUTs \
165183
-DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
166184
-DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
185+
-Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
186+
-DfailIfNoTests=false \
187+
-Dfailsafe.failIfNoSpecifiedTests=false \
167188
-pl integration-test \
168189
-am -PMultiClusterIT2DualTreeAutoBasic \
169190
-ntp >> ~/run-tests-$attempt.log && return 0
@@ -201,9 +222,11 @@ jobs:
201222
if: failure()
202223
uses: actions/upload-artifact@v6
203224
with:
204-
name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
225+
name: cluster-log-dual-tree-auto-basic-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
205226
path: integration-test/target/cluster-logs
206227
retention-days: 30
228+
# 9 IT classes split across 3 parallel shards to cut the historical ~51 min
229+
# wall clock to ~17 min. See cluster-it-1c1d.yml for the shard pattern.
207230
dual-tree-auto-enhanced:
208231
strategy:
209232
fail-fast: false
@@ -214,6 +237,7 @@ jobs:
214237
cluster1: [HighPerformanceMode]
215238
cluster2: [HighPerformanceMode]
216239
os: [ubuntu-latest]
240+
shard: [0, 1, 2]
217241
runs-on: ${{ matrix.os }}
218242
steps:
219243
- uses: actions/checkout@v5
@@ -233,6 +257,21 @@ jobs:
233257
- name: Sleep for a random duration between 0 and 10000 milliseconds
234258
run: |
235259
sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
260+
- name: Build IT shard list
261+
shell: bash
262+
# See cluster-it-1c1d.yml for the shard-list pattern. Write under
263+
# $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
264+
run: |
265+
set -euo pipefail
266+
SHARD=${{ matrix.shard }}
267+
TOTAL=3
268+
grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTreeAutoEnhanced\b' integration-test/src/test/java \
269+
| awk -F'/' '{print $NF}' | sed 's/\.java$//' \
270+
| sort \
271+
| awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
272+
> "$RUNNER_TEMP/it-shard.txt"
273+
echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes"
274+
head -5 "$RUNNER_TEMP/it-shard.txt"
236275
- name: IT Test
237276
shell: bash
238277
# we do not compile client-cpp for saving time, it is tested in client.yml
@@ -250,6 +289,9 @@ jobs:
250289
-DskipUTs \
251290
-DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
252291
-DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
292+
-Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
293+
-DfailIfNoTests=false \
294+
-Dfailsafe.failIfNoSpecifiedTests=false \
253295
-pl integration-test \
254296
-am -PMultiClusterIT2DualTreeAutoEnhanced \
255297
-ntp >> ~/run-tests-$attempt.log && return 0
@@ -287,9 +329,11 @@ jobs:
287329
if: failure()
288330
uses: actions/upload-artifact@v6
289331
with:
290-
name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
332+
name: cluster-log-dual-tree-auto-enhanced-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
291333
path: integration-test/target/cluster-logs
292334
retention-days: 30
335+
# 11 IT classes split across 3 parallel shards to cut the historical ~27 min
336+
# wall clock to ~9 min. See cluster-it-1c1d.yml for the shard pattern.
293337
dual-tree-manual:
294338
strategy:
295339
fail-fast: false
@@ -300,6 +344,7 @@ jobs:
300344
cluster1: [HighPerformanceMode]
301345
cluster2: [HighPerformanceMode]
302346
os: [ubuntu-latest]
347+
shard: [0, 1, 2]
303348
runs-on: ${{ matrix.os }}
304349
steps:
305350
- uses: actions/checkout@v5
@@ -319,6 +364,21 @@ jobs:
319364
- name: Sleep for a random duration between 0 and 10000 milliseconds
320365
run: |
321366
sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
367+
- name: Build IT shard list
368+
shell: bash
369+
# See cluster-it-1c1d.yml for the shard-list pattern. Write under
370+
# $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
371+
run: |
372+
set -euo pipefail
373+
SHARD=${{ matrix.shard }}
374+
TOTAL=3
375+
grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTreeManual\b' integration-test/src/test/java \
376+
| awk -F'/' '{print $NF}' | sed 's/\.java$//' \
377+
| sort \
378+
| awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
379+
> "$RUNNER_TEMP/it-shard.txt"
380+
echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes"
381+
head -5 "$RUNNER_TEMP/it-shard.txt"
322382
- name: IT Test
323383
shell: bash
324384
# we do not compile client-cpp for saving time, it is tested in client.yml
@@ -336,6 +396,9 @@ jobs:
336396
-DskipUTs \
337397
-DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
338398
-DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
399+
-Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
400+
-DfailIfNoTests=false \
401+
-Dfailsafe.failIfNoSpecifiedTests=false \
339402
-pl integration-test \
340403
-am -PMultiClusterIT2DualTreeManual \
341404
-ntp >> ~/run-tests-$attempt.log && return 0
@@ -373,7 +436,7 @@ jobs:
373436
if: failure()
374437
uses: actions/upload-artifact@v6
375438
with:
376-
name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
439+
name: cluster-log-dual-tree-manual-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
377440
path: integration-test/target/cluster-logs
378441
retention-days: 30
379442
subscription-tree-arch-verification:
@@ -720,6 +783,8 @@ jobs:
720783
name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
721784
path: integration-test/target/cluster-logs
722785
retention-days: 30
786+
# 13 IT classes split across 3 parallel shards to cut the historical ~63 min
787+
# wall clock to ~22 min. See cluster-it-1c1d.yml for the shard pattern.
723788
dual-table-manual-basic:
724789
strategy:
725790
fail-fast: false
@@ -729,6 +794,7 @@ jobs:
729794
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
730795
cluster: [HighPerformanceMode]
731796
os: [ubuntu-latest]
797+
shard: [0, 1, 2]
732798
runs-on: ${{ matrix.os }}
733799
steps:
734800
- uses: actions/checkout@v5
@@ -748,6 +814,21 @@ jobs:
748814
- name: Sleep for a random duration between 0 and 10000 milliseconds
749815
run: |
750816
sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
817+
- name: Build IT shard list
818+
shell: bash
819+
# See cluster-it-1c1d.yml for the shard-list pattern. Write under
820+
# $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
821+
run: |
822+
set -euo pipefail
823+
SHARD=${{ matrix.shard }}
824+
TOTAL=3
825+
grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTableManualBasic\b' integration-test/src/test/java \
826+
| awk -F'/' '{print $NF}' | sed 's/\.java$//' \
827+
| sort \
828+
| awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
829+
> "$RUNNER_TEMP/it-shard.txt"
830+
echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes"
831+
head -5 "$RUNNER_TEMP/it-shard.txt"
751832
- name: IT Test
752833
shell: bash
753834
# we do not compile client-cpp for saving time, it is tested in client.yml
@@ -765,6 +846,9 @@ jobs:
765846
-DskipUTs \
766847
-DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
767848
-DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
849+
-Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
850+
-DfailIfNoTests=false \
851+
-Dfailsafe.failIfNoSpecifiedTests=false \
768852
-pl integration-test \
769853
-am -PMultiClusterIT2DualTableManualBasic \
770854
-ntp >> ~/run-tests-$attempt.log && return 0
@@ -802,9 +886,11 @@ jobs:
802886
if: failure()
803887
uses: actions/upload-artifact@v6
804888
with:
805-
name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
889+
name: cluster-log-dual-table-manual-basic-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
806890
path: integration-test/target/cluster-logs
807891
retention-days: 30
892+
# 11 IT classes split across 3 parallel shards to cut the historical ~62 min
893+
# wall clock to ~22 min. See cluster-it-1c1d.yml for the shard pattern.
808894
dual-table-manual-enhanced:
809895
strategy:
810896
fail-fast: false
@@ -814,6 +900,7 @@ jobs:
814900
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
815901
cluster: [HighPerformanceMode]
816902
os: [ubuntu-latest]
903+
shard: [0, 1, 2]
817904
runs-on: ${{ matrix.os }}
818905
steps:
819906
- uses: actions/checkout@v5
@@ -833,6 +920,21 @@ jobs:
833920
- name: Sleep for a random duration between 0 and 10000 milliseconds
834921
run: |
835922
sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
923+
- name: Build IT shard list
924+
shell: bash
925+
# See cluster-it-1c1d.yml for the shard-list pattern. Write under
926+
# $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file.
927+
run: |
928+
set -euo pipefail
929+
SHARD=${{ matrix.shard }}
930+
TOTAL=3
931+
grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTableManualEnhanced\b' integration-test/src/test/java \
932+
| awk -F'/' '{print $NF}' | sed 's/\.java$//' \
933+
| sort \
934+
| awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
935+
> "$RUNNER_TEMP/it-shard.txt"
936+
echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes"
937+
head -5 "$RUNNER_TEMP/it-shard.txt"
836938
- name: IT Test
837939
shell: bash
838940
# we do not compile client-cpp for saving time, it is tested in client.yml
@@ -850,6 +952,9 @@ jobs:
850952
-DskipUTs \
851953
-DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
852954
-DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
955+
-Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
956+
-DfailIfNoTests=false \
957+
-Dfailsafe.failIfNoSpecifiedTests=false \
853958
-pl integration-test \
854959
-am -PMultiClusterIT2DualTableManualEnhanced \
855960
-ntp >> ~/run-tests-$attempt.log && return 0
@@ -887,7 +992,7 @@ jobs:
887992
if: failure()
888993
uses: actions/upload-artifact@v6
889994
with:
890-
name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
995+
name: cluster-log-dual-table-manual-enhanced-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
891996
path: integration-test/target/cluster-logs
892997
retention-days: 30
893998
triple:

0 commit comments

Comments
 (0)