Skip to content

Commit 2b08a74

Browse files
temblekingfcracker79
tembleking
authored and
fcracker79
committed
ci(test): parallelize acceptance tests using dynamic matrix (#692)
## Summary Parallelize Sysdig acceptance tests by running each test file in its own GitHub Actions job using dynamic matrix strategy. **IBM tests remain sequential due to API rate limiting issues.** ## Changes | Suite | Before | After | |-------|--------|-------| | Sysdig Secure | 1 sequential job | Dynamic matrix (max 20 concurrent), fail-fast | | Sysdig Monitor | 1 sequential job | Dynamic matrix (max 20 concurrent), fail-fast | | IBM Monitor | 1 sequential job | No change (sequential) | | IBM Secure | 1 sequential job | No change (sequential) | ## How it works 1. `list-*-tests` jobs discover test files by searching for build tags (`tf_acc_sysdig_secure`, `tf_acc_sysdig_monitor`) 2. Each file's tests run in parallel using `-run` flag to filter by test name 3. `fail-fast: true` stops the matrix early if any test fails 4. Aggregator jobs (`sysdig-secure-result`, `sysdig-monitor-result`) collect results for required status checks 5. IBM tests remain sequential to avoid API rate limiting (500/504 errors with parallelization) ## Additional changes - Add `merge_group` trigger to `ci.yml` for merge queue support - Handle empty grep results gracefully in list jobs - Fix `data_source_sysdig_user_test` to use random email suffix (avoid collisions in parallel runs) - Fix `data_source_sysdig_secure_rule_stateful_count_test` to check `rule_count >= 2` instead of exact match (avoid flaky failures)
1 parent 066a36b commit 2b08a74

4 files changed

Lines changed: 146 additions & 39 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ on:
77
pull_request:
88
branches:
99
- master
10+
merge_group:
11+
branches:
12+
- master
1013

1114
jobs:
1215
build-multiarch:

‎.github/workflows/test.yml‎

Lines changed: 114 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,9 @@ jobs:
77
lint:
88
name: Lint
99
runs-on: ubuntu-latest
10-
1110
steps:
1211
- name: Check out code
1312
uses: actions/checkout@v4
14-
1513
- name: Lint
1614
uses: golangci/golangci-lint-action@v8
1715
with:
@@ -20,75 +18,158 @@ jobs:
2018
test:
2119
name: Unit Tests
2220
runs-on: ubuntu-latest
23-
2421
steps:
2522
- name: Check out code
2623
uses: actions/checkout@v4
27-
2824
- name: Set up Go
2925
uses: actions/setup-go@v5
3026
with:
3127
go-version-file: go.mod
32-
3328
- name: Test
3429
run: make test
3530

36-
test-sysdig-monitor:
37-
name: Sysdig Monitor Acceptance Tests
31+
# ============================================
32+
# Sysdig Secure - Dynamic Matrix
33+
# ============================================
34+
list-sysdig-secure-tests:
35+
name: List Secure Tests
3836
runs-on: ubuntu-latest
3937
needs: test
40-
38+
outputs:
39+
matrix: ${{ steps.list.outputs.matrix }}
4140
steps:
42-
- name: Check out code
43-
uses: actions/checkout@v4
41+
- uses: actions/checkout@v4
42+
- id: list
43+
run: |
44+
files=$(grep -l "tf_acc_sysdig_secure" sysdig/*_test.go | xargs -I{} basename {} .go || echo "")
45+
if [ -z "$files" ]; then
46+
echo "matrix=[]" >> $GITHUB_OUTPUT
47+
else
48+
matrix=$(echo "$files" | jq -R -s -c 'split("\n") | map(select(length > 0))')
49+
echo "matrix=$matrix" >> $GITHUB_OUTPUT
50+
fi
4451
45-
- name: Set up Go
46-
uses: actions/setup-go@v5
52+
test-sysdig-secure:
53+
name: "Secure: ${{ matrix.file }}"
54+
runs-on: ubuntu-latest
55+
needs: list-sysdig-secure-tests
56+
strategy:
57+
fail-fast: true
58+
max-parallel: 20
59+
matrix:
60+
file: ${{ fromJson(needs.list-sysdig-secure-tests.outputs.matrix) }}
61+
steps:
62+
- uses: actions/checkout@v4
63+
- uses: actions/setup-go@v5
4764
with:
4865
go-version-file: go.mod
49-
50-
- name: Test
51-
run: make testacc
66+
- name: Run tests from ${{ matrix.file }}
67+
run: |
68+
tests=$(grep -oh "func Test[A-Za-z0-9_]*" sysdig/${{ matrix.file }}.go | sed 's/func //' | tr '\n' '|' | sed 's/|$//')
69+
if [ -n "$tests" ]; then
70+
echo "Running tests: $tests"
71+
CGO_ENABLED=1 TF_ACC=1 go test ./sysdig -v -tags=tf_acc_sysdig_secure -timeout 30m -race -run "^($tests)$"
72+
else
73+
echo "No tests found in file"
74+
fi
5275
env:
53-
TEST_SUITE: tf_acc_sysdig_monitor
5476
SYSDIG_MONITOR_API_TOKEN: ${{ secrets.KUBELAB_MONITOR_API_TOKEN }}
5577
SYSDIG_SECURE_API_TOKEN: ${{ secrets.KUBELAB_SECURE_API_TOKEN }}
5678

57-
test-sysdig-secure:
58-
name: Sysdig Secure Acceptance Tests
79+
sysdig-secure-result:
80+
name: Sysdig Secure Tests Result
81+
runs-on: ubuntu-latest
82+
needs: [list-sysdig-secure-tests, test-sysdig-secure]
83+
if: always()
84+
steps:
85+
- name: Check test results
86+
run: |
87+
if [ "${{ needs.test-sysdig-secure.result }}" == "success" ] || [ "${{ needs.test-sysdig-secure.result }}" == "skipped" ]; then
88+
echo "All Sysdig Secure tests passed"
89+
exit 0
90+
else
91+
echo "Some Sysdig Secure tests failed"
92+
exit 1
93+
fi
94+
95+
# ============================================
96+
# Sysdig Monitor - Dynamic Matrix
97+
# ============================================
98+
list-sysdig-monitor-tests:
99+
name: List Monitor Tests
59100
runs-on: ubuntu-latest
60101
needs: test
61-
102+
outputs:
103+
matrix: ${{ steps.list.outputs.matrix }}
62104
steps:
63-
- name: Check out code
64-
uses: actions/checkout@v4
105+
- uses: actions/checkout@v4
106+
- id: list
107+
run: |
108+
files=$(grep -l "tf_acc_sysdig_monitor" sysdig/*_test.go | xargs -I{} basename {} .go || echo "")
109+
if [ -z "$files" ]; then
110+
echo "matrix=[]" >> $GITHUB_OUTPUT
111+
else
112+
matrix=$(echo "$files" | jq -R -s -c 'split("\n") | map(select(length > 0))')
113+
echo "matrix=$matrix" >> $GITHUB_OUTPUT
114+
fi
65115
66-
- name: Set up Go
67-
uses: actions/setup-go@v5
116+
test-sysdig-monitor:
117+
name: "Monitor: ${{ matrix.file }}"
118+
runs-on: ubuntu-latest
119+
needs: list-sysdig-monitor-tests
120+
strategy:
121+
fail-fast: true
122+
max-parallel: 20
123+
matrix:
124+
file: ${{ fromJson(needs.list-sysdig-monitor-tests.outputs.matrix) }}
125+
steps:
126+
- uses: actions/checkout@v4
127+
- uses: actions/setup-go@v5
68128
with:
69129
go-version-file: go.mod
70-
71-
- name: Test
72-
run: make testacc
130+
- name: Run tests from ${{ matrix.file }}
131+
run: |
132+
tests=$(grep -oh "func Test[A-Za-z0-9_]*" sysdig/${{ matrix.file }}.go | sed 's/func //' | tr '\n' '|' | sed 's/|$//')
133+
if [ -n "$tests" ]; then
134+
echo "Running tests: $tests"
135+
CGO_ENABLED=1 TF_ACC=1 go test ./sysdig -v -tags=tf_acc_sysdig_monitor -timeout 30m -race -run "^($tests)$"
136+
else
137+
echo "No tests found in file"
138+
fi
73139
env:
74-
TEST_SUITE: tf_acc_sysdig_secure
75140
SYSDIG_MONITOR_API_TOKEN: ${{ secrets.KUBELAB_MONITOR_API_TOKEN }}
76141
SYSDIG_SECURE_API_TOKEN: ${{ secrets.KUBELAB_SECURE_API_TOKEN }}
77142

143+
sysdig-monitor-result:
144+
name: Sysdig Monitor Tests Result
145+
runs-on: ubuntu-latest
146+
needs: [list-sysdig-monitor-tests, test-sysdig-monitor]
147+
if: always()
148+
steps:
149+
- name: Check test results
150+
run: |
151+
if [ "${{ needs.test-sysdig-monitor.result }}" == "success" ] || [ "${{ needs.test-sysdig-monitor.result }}" == "skipped" ]; then
152+
echo "All Sysdig Monitor tests passed"
153+
exit 0
154+
else
155+
echo "Some Sysdig Monitor tests failed"
156+
exit 1
157+
fi
158+
159+
# ============================================
160+
# IBM Monitor - Sequential (no parallelization)
161+
# ============================================
78162
test-ibm-monitor:
79163
name: IBM Monitor Acceptance Tests
80164
runs-on: ubuntu-latest
81165
needs: test
82-
83166
steps:
84167
- name: Check out code
85168
uses: actions/checkout@v4
86-
87169
- name: Set up Go
88170
uses: actions/setup-go@v5
89171
with:
90172
go-version-file: go.mod
91-
92173
- name: Test
93174
run: make testacc
94175
env:
@@ -99,20 +180,20 @@ jobs:
99180
SYSDIG_MONITOR_URL: "https://eu-gb.monitoring.cloud.ibm.com"
100181
IBM_EVENT_NOTIFICATION_INSTANCE_ID: ${{ secrets.IBM_EVENT_NOTIFICATION_INSTANCE_ID }}
101182

183+
# ============================================
184+
# IBM Secure - Sequential (no parallelization)
185+
# ============================================
102186
test-ibm-secure:
103187
name: IBM Secure Acceptance Tests
104188
runs-on: ubuntu-latest
105189
needs: test
106-
107190
steps:
108191
- name: Check out code
109192
uses: actions/checkout@v4
110-
111193
- name: Set up Go
112194
uses: actions/setup-go@v5
113195
with:
114196
go-version-file: go.mod
115-
116197
- name: Test
117198
run: make testacc
118199
env:

‎sysdig/data_source_sysdig_secure_rule_stateful_count_test.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ package sysdig_test
55
import (
66
"fmt"
77
"os"
8+
"strconv"
89
"strings"
910
"testing"
1011

1112
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
1213
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
14+
"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
1315

1416
"github.com/draios/terraform-provider-sysdig/sysdig"
1517
)
@@ -35,13 +37,31 @@ func TestAccRuleStatefulCountDataSource(t *testing.T) {
3537
{
3638
Config: ruleStatefulCountDataSource(),
3739
Check: resource.ComposeTestCheckFunc(
38-
resource.TestCheckResourceAttr("data.sysdig_secure_rule_stateful_count.data_stateful_rule_append", "rule_count", "2"),
40+
testCheckRuleCountAtLeast("data.sysdig_secure_rule_stateful_count.data_stateful_rule_append", 2),
3941
),
4042
},
4143
},
4244
})
4345
}
4446

47+
func testCheckRuleCountAtLeast(resourceName string, minCount int) resource.TestCheckFunc {
48+
return func(s *terraform.State) error {
49+
rs, ok := s.RootModule().Resources[resourceName]
50+
if !ok {
51+
return fmt.Errorf("resource not found: %s", resourceName)
52+
}
53+
countStr := rs.Primary.Attributes["rule_count"]
54+
count, err := strconv.Atoi(countStr)
55+
if err != nil {
56+
return fmt.Errorf("rule_count is not a valid integer: %s", countStr)
57+
}
58+
if count < minCount {
59+
return fmt.Errorf("rule_count expected >= %d, got %d", minCount, count)
60+
}
61+
return nil
62+
}
63+
}
64+
4565
func ruleStatefulCountDataSource() string {
4666
return fmt.Sprintf(`
4767
%s

‎sysdig/data_source_sysdig_user_test.go‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,18 @@
33
package sysdig_test
44

55
import (
6+
"fmt"
67
"testing"
78

9+
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
810
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
911
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
1012

1113
"github.com/draios/terraform-provider-sysdig/sysdig"
1214
)
1315

1416
func TestAccDataUser(t *testing.T) {
17+
randomSuffix := acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum)
1518
resource.ParallelTest(t, resource.TestCase{
1619
PreCheck: preCheckAnyEnv(t, SysdigMonitorApiTokenEnv, SysdigSecureApiTokenEnv),
1720
ProviderFactories: map[string]func() (*schema.Provider, error){
@@ -21,21 +24,21 @@ func TestAccDataUser(t *testing.T) {
2124
},
2225
Steps: []resource.TestStep{
2326
{
24-
Config: getUser(),
27+
Config: getUser(randomSuffix),
2528
},
2629
},
2730
})
2831
}
2932

30-
func getUser() string {
31-
return `
33+
func getUser(suffix string) string {
34+
return fmt.Sprintf(`
3235
resource "sysdig_user" "sample" {
33-
email = "terraform-test+user@sysdig.com"
36+
email = "terraform-test+user-%s@sysdig.com"
3437
}
3538
3639
data "sysdig_user" "me" {
3740
depends_on = ["sysdig_user.sample"]
3841
email = sysdig_user.sample.email
3942
}
40-
`
43+
`, suffix)
4144
}

0 commit comments

Comments
 (0)