Skip to content

Commit cc23012

Browse files
rhmdndrhmdnd
committed
Use t.Error instead of t.Fatal for assertion checking
The tests will fail if there are any discrepancies between the expected cluster state and the observed state. Using t.Fataf() or t.Fatal marks the test as failed, which is what we want, but it also stop execution of the test. Since we have some helper methods that process the mismatched results to generate assertion files and reports, we don't want to completely stop execution so that we can still produce those files. This commit updates those specific parts of the tests to use Error instead of Fatal so we perserve the behavior to mark the test as failed, but continue processing in the test to cleanup and generate reports.
1 parent def3a79 commit cc23012

File tree

4 files changed

+181
-4
lines changed

4 files changed

+181
-4
lines changed

‎e2e_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ func TestPlatformCompliance(t *testing.T) {
169169
if err = helpers.GenerateMismatchReport(tc, c, mismatchedAssertions, platformBindingName); err != nil {
170170
t.Fatalf("Failed to generate test report: %s", err)
171171
}
172-
t.Fatal("Actual cluster compliance state didn't match expected state")
172+
t.Error("Actual cluster compliance state didn't match expected state")
173173
}
174174

175175
err = helpers.GenerateAssertionFileFromResults(tc, c, assertionFileName, initialResults, finalResults)
@@ -282,7 +282,7 @@ func TestNodeCompliance(t *testing.T) {
282282
if err = helpers.GenerateMismatchReport(tc, c, mismatchedAssertions, nodeBindingName); err != nil {
283283
t.Fatalf("Failed to generate test report: %s", err)
284284
}
285-
t.Fatal("Actual cluster compliance state didn't match expected state")
285+
t.Error("Actual cluster compliance state didn't match expected state")
286286
}
287287

288288
err = helpers.GenerateAssertionFileFromResults(tc, c, assertionFileName, initialResults, finalResults)
@@ -352,7 +352,7 @@ func TestProfile(t *testing.T) {
352352
if err = helpers.GenerateMismatchReport(tc, c, mismatchedAssertions, bindingName); err != nil {
353353
t.Fatalf("Failed to generate test report: %s", err)
354354
}
355-
t.Fatal("Actual cluster compliance state didn't match expected state")
355+
t.Error("Actual cluster compliance state didn't match expected state")
356356
}
357357

358358
err = helpers.GenerateAssertionFileFromResults(tc, c, assertionFileName, initialResults, nil)
@@ -477,7 +477,7 @@ func TestProfileRemediations(t *testing.T) {
477477
if err = helpers.GenerateMismatchReport(tc, c, mismatchedAssertions, bindingName); err != nil {
478478
t.Fatalf("Failed to generate test report: %s", err)
479479
}
480-
t.Fatal("Actual cluster compliance state didn't match expected state")
480+
t.Error("Actual cluster compliance state didn't match expected state")
481481
}
482482

483483
err = helpers.GenerateAssertionFileFromResults(tc, c, assertionFileName, initialResults, finalResults)

‎logs/artifacts/iniital-ocp4-cis-mismatches.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
- check_result_name: ocp4-cis-api-server-insecure-port
2+
expected_result: NOT-APPLICABLE
3+
actual_result: ""
4+
error_message: 'E2E-FAILURE: Expected to find rule ocp4-cis-api-server-insecure-port
5+
in scan results with NOT-APPLICABLE state'
6+
- check_result_name: ocp4-cis-configure-network-policies-hypershift-hosted
7+
expected_result: NOT-APPLICABLE
8+
actual_result: ""
9+
error_message: 'E2E-FAILURE: Expected to find rule ocp4-cis-configure-network-policies-hypershift-hosted
10+
in scan results with NOT-APPLICABLE state'
11+
- check_result_name: ocp4-cis-api-server-tls-cipher-suites
12+
expected_result: PASS
13+
actual_result: ""
14+
error_message: 'E2E-FAILURE: Expected to find rule ocp4-cis-api-server-tls-cipher-suites
15+
in scan results with PASS state'
16+
- check_result_name: ocp4-cis-audit-log-forwarding-webhook
17+
expected_result: NOT-APPLICABLE
18+
actual_result: ""
19+
error_message: 'E2E-FAILURE: Expected to find rule ocp4-cis-audit-log-forwarding-webhook
20+
in scan results with NOT-APPLICABLE state'
21+
- check_result_name: ocp4-cis-file-permissions-proxy-kubeconfig
22+
expected_result: NOT-APPLICABLE
23+
actual_result: ""
24+
error_message: 'E2E-FAILURE: Expected to find rule ocp4-cis-file-permissions-proxy-kubeconfig
25+
in scan results with NOT-APPLICABLE state'
26+
- check_result_name: ocp4-cis-api-server-api-priority-gate-enabled
27+
expected_result: NOT-APPLICABLE
28+
actual_result: ""
29+
error_message: 'E2E-FAILURE: Expected to find rule ocp4-cis-api-server-api-priority-gate-enabled
30+
in scan results with NOT-APPLICABLE state'
31+
- check_result_name: ocp4-cis-file-groupowner-proxy-kubeconfig
32+
expected_result: NOT-APPLICABLE
33+
actual_result: ""
34+
error_message: 'E2E-FAILURE: Expected to find rule ocp4-cis-file-groupowner-proxy-kubeconfig
35+
in scan results with NOT-APPLICABLE state'
36+
- check_result_name: ocp4-cis-file-owner-proxy-kubeconfig
37+
expected_result: NOT-APPLICABLE
38+
actual_result: ""
39+
error_message: 'E2E-FAILURE: Expected to find rule ocp4-cis-file-owner-proxy-kubeconfig
40+
in scan results with NOT-APPLICABLE state'

‎logs/artifacts/ocp4-cis-test-binding-report.html‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
<title>Compliance Report</title>
5+
<style>
6+
body { font-family: Arial, sans-serif; margin: 20px; }
7+
h1, h2, h3 { color: #333; }
8+
code { background-color: #f4f4f4; padding: 2px 4px; }
9+
li { margin: 5px 0; }
10+
</style>
11+
</head>
12+
<body>
13+
<h1>ocp4-cis-test-binding Compliance Test Results</h1>
14+
15+
<strong>Test Run Date:</strong> 2025-10-03 09:40:05 UTC
16+
<strong>Platform:</strong> ocp4
17+
<strong>Version:</strong> 4.18
18+
<strong>Content Image:</strong> quay.io/redhat-user-workloads/ocp-isc-tenant/compliance-operator-content-dev:master
19+
20+
<h2>Summary</h2>
21+
22+
<strong>Total Assertion Failures:</strong> 8
23+
24+
<h2>Detailed Failures</h2>
25+
26+
<h3>1. ocp4-cis-api-server-insecure-port</h3>
27+
28+
<li><strong>Expected Result:</strong> <code>NOT-APPLICABLE</code></li>
29+
<li><strong>Actual Result:</strong> ``</li>
30+
<li><strong>Error Details:</strong> E2E-FAILURE: Expected to find rule ocp4-cis-api-server-insecure-port in scan results with NOT-APPLICABLE state</li>
31+
32+
<h3>2. ocp4-cis-configure-network-policies-hypershift-hosted</h3>
33+
34+
<li><strong>Expected Result:</strong> <code>NOT-APPLICABLE</code></li>
35+
<li><strong>Actual Result:</strong> ``</li>
36+
<li><strong>Error Details:</strong> E2E-FAILURE: Expected to find rule ocp4-cis-configure-network-policies-hypershift-hosted in scan results with NOT-APPLICABLE state</li>
37+
38+
<h3>3. ocp4-cis-api-server-tls-cipher-suites</h3>
39+
40+
<li><strong>Expected Result:</strong> <code>PASS</code></li>
41+
<li><strong>Actual Result:</strong> ``</li>
42+
<li><strong>Error Details:</strong> E2E-FAILURE: Expected to find rule ocp4-cis-api-server-tls-cipher-suites in scan results with PASS state</li>
43+
44+
<h3>4. ocp4-cis-audit-log-forwarding-webhook</h3>
45+
46+
<li><strong>Expected Result:</strong> <code>NOT-APPLICABLE</code></li>
47+
<li><strong>Actual Result:</strong> ``</li>
48+
<li><strong>Error Details:</strong> E2E-FAILURE: Expected to find rule ocp4-cis-audit-log-forwarding-webhook in scan results with NOT-APPLICABLE state</li>
49+
50+
<h3>5. ocp4-cis-file-permissions-proxy-kubeconfig</h3>
51+
52+
<li><strong>Expected Result:</strong> <code>NOT-APPLICABLE</code></li>
53+
<li><strong>Actual Result:</strong> ``</li>
54+
<li><strong>Error Details:</strong> E2E-FAILURE: Expected to find rule ocp4-cis-file-permissions-proxy-kubeconfig in scan results with NOT-APPLICABLE state</li>
55+
56+
<h3>6. ocp4-cis-api-server-api-priority-gate-enabled</h3>
57+
58+
<li><strong>Expected Result:</strong> <code>NOT-APPLICABLE</code></li>
59+
<li><strong>Actual Result:</strong> ``</li>
60+
<li><strong>Error Details:</strong> E2E-FAILURE: Expected to find rule ocp4-cis-api-server-api-priority-gate-enabled in scan results with NOT-APPLICABLE state</li>
61+
62+
<h3>7. ocp4-cis-file-groupowner-proxy-kubeconfig</h3>
63+
64+
<li><strong>Expected Result:</strong> <code>NOT-APPLICABLE</code></li>
65+
<li><strong>Actual Result:</strong> ``</li>
66+
<li><strong>Error Details:</strong> E2E-FAILURE: Expected to find rule ocp4-cis-file-groupowner-proxy-kubeconfig in scan results with NOT-APPLICABLE state</li>
67+
68+
<h3>8. ocp4-cis-file-owner-proxy-kubeconfig</h3>
69+
70+
<li><strong>Expected Result:</strong> <code>NOT-APPLICABLE</code></li>
71+
<li><strong>Actual Result:</strong> ``</li>
72+
<li><strong>Error Details:</strong> E2E-FAILURE: Expected to find rule ocp4-cis-file-owner-proxy-kubeconfig in scan results with NOT-APPLICABLE state</li>
73+
74+
75+
</body>
76+
</html>

‎logs/artifacts/ocp4-cis-test-binding-report.md‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# ocp4-cis-test-binding Compliance Test Results
2+
3+
**Test Run Date:** 2025-10-03 09:40:05 UTC
4+
**Platform:** ocp4
5+
**Version:** 4.18
6+
**Content Image:** quay.io/redhat-user-workloads/ocp-isc-tenant/compliance-operator-content-dev:master
7+
8+
## Summary
9+
10+
**Total Assertion Failures:** 8
11+
12+
## Detailed Failures
13+
14+
### 1. ocp4-cis-api-server-insecure-port
15+
16+
- **Expected Result:** `NOT-APPLICABLE`
17+
- **Actual Result:** ``
18+
- **Error Details:** E2E-FAILURE: Expected to find rule ocp4-cis-api-server-insecure-port in scan results with NOT-APPLICABLE state
19+
20+
### 2. ocp4-cis-configure-network-policies-hypershift-hosted
21+
22+
- **Expected Result:** `NOT-APPLICABLE`
23+
- **Actual Result:** ``
24+
- **Error Details:** E2E-FAILURE: Expected to find rule ocp4-cis-configure-network-policies-hypershift-hosted in scan results with NOT-APPLICABLE state
25+
26+
### 3. ocp4-cis-api-server-tls-cipher-suites
27+
28+
- **Expected Result:** `PASS`
29+
- **Actual Result:** ``
30+
- **Error Details:** E2E-FAILURE: Expected to find rule ocp4-cis-api-server-tls-cipher-suites in scan results with PASS state
31+
32+
### 4. ocp4-cis-audit-log-forwarding-webhook
33+
34+
- **Expected Result:** `NOT-APPLICABLE`
35+
- **Actual Result:** ``
36+
- **Error Details:** E2E-FAILURE: Expected to find rule ocp4-cis-audit-log-forwarding-webhook in scan results with NOT-APPLICABLE state
37+
38+
### 5. ocp4-cis-file-permissions-proxy-kubeconfig
39+
40+
- **Expected Result:** `NOT-APPLICABLE`
41+
- **Actual Result:** ``
42+
- **Error Details:** E2E-FAILURE: Expected to find rule ocp4-cis-file-permissions-proxy-kubeconfig in scan results with NOT-APPLICABLE state
43+
44+
### 6. ocp4-cis-api-server-api-priority-gate-enabled
45+
46+
- **Expected Result:** `NOT-APPLICABLE`
47+
- **Actual Result:** ``
48+
- **Error Details:** E2E-FAILURE: Expected to find rule ocp4-cis-api-server-api-priority-gate-enabled in scan results with NOT-APPLICABLE state
49+
50+
### 7. ocp4-cis-file-groupowner-proxy-kubeconfig
51+
52+
- **Expected Result:** `NOT-APPLICABLE`
53+
- **Actual Result:** ``
54+
- **Error Details:** E2E-FAILURE: Expected to find rule ocp4-cis-file-groupowner-proxy-kubeconfig in scan results with NOT-APPLICABLE state
55+
56+
### 8. ocp4-cis-file-owner-proxy-kubeconfig
57+
58+
- **Expected Result:** `NOT-APPLICABLE`
59+
- **Actual Result:** ``
60+
- **Error Details:** E2E-FAILURE: Expected to find rule ocp4-cis-file-owner-proxy-kubeconfig in scan results with NOT-APPLICABLE state
61+

0 commit comments

Comments
 (0)