iotdb/.github/workflows/table-cluster-it-1c1d.yml at dab45fb2325685cc4cde21b33115e4eb3fc48cfb · apache/iotdb · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
name: Table Cluster IT - 1C1D

on:
  push:
    branches:
      - master
      - "rel/*"
      - "rc/*"
    paths-ignore:
      - "docs/**"
      - "site/**"
  pull_request:
    branches:
      - master
      - "rel/*"
      - "rc/*"
      - "force_ci/**"
    paths-ignore:
      - "docs/**"
      - "site/**"
  # allow manually run the action:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
  MAVEN_ARGS: --batch-mode --no-transfer-progress
  DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}

jobs:
  # Ubuntu runs all ITs in a single job (already fast at ~39 min)
  Ubuntu:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: 17
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Adjust Linux kernel somaxconn
        shell: bash
        run: sudo sysctl -w net.core.somaxconn=65535
      - name: IT/UT Test
        shell: bash
        run: |
          mvn clean verify \
          -P with-integration-tests \
          -DskipUTs \
          -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 \
          -pl integration-test \
          -am -PTableSimpleIT
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: table-standalone-log-Linux
          path: integration-test/target/cluster-logs
          retention-days: 1

  # Windows is ~67% slower than Ubuntu, so split into 3 shards to parallelize
  Windows:
    strategy:
      fail-fast: false
      matrix:
        shard: [0, 1, 2]
    runs-on: windows-latest

    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: 17
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Adjust network dynamic TCP ports range
        shell: pwsh
        run: |
          netsh int ipv4 set dynamicport tcp start=32768 num=32768
          netsh int ipv4 set dynamicport udp start=32768 num=32768
          netsh int ipv6 set dynamicport tcp start=32768 num=32768
          netsh int ipv6 set dynamicport udp start=32768 num=32768
      - name: Build IT shard list
        shell: bash
        # Distribute TableLocalStandaloneIT test classes across 3 shards using hash-mod assignment.
        # The list is written to a file so failsafe.includesFile can read it without command-line length limits.
        run: |
          set -euo pipefail
          SHARD=${{ matrix.shard }}
          TOTAL=3
          # Write outside the repo so Apache RAT (license check) doesn't flag the file.
          # Using a single grep -rl call instead of `find | xargs grep`: on Windows Git Bash,
          # ARG_MAX is small so xargs batches the file list, and any batch with no matches
          # makes grep exit 1, which makes xargs exit 123 and trips `set -o pipefail`.
          grep -rl --include='*IT.java' 'TableLocalStandaloneIT' integration-test/src/test/java \
            | awk -F'/' '{print $NF}' | sed 's/\.java$//' \
            | sort \
            | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \
            > "$RUNNER_TEMP/it-shard.txt"
          echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes"
          head -5 "$RUNNER_TEMP/it-shard.txt"
      - name: IT/UT Test
        shell: bash
        run: |
          mvn clean verify \
          -P with-integration-tests \
          -DskipUTs \
          -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 \
          -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \
          -DfailIfNoTests=false \
          -Dfailsafe.failIfNoSpecifiedTests=false \
          -pl integration-test \
          -am -PTableSimpleIT
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: table-standalone-log-Windows-shard${{ matrix.shard }}
          path: integration-test/target/cluster-logs
          retention-days: 1