Skip to content

Commit f8d0feb

Browse files
dangrondahldangrondahl
committed
green: backfill wantExitCode annotations across assert/attest/evaluate tests
Add wantExitCode to 44 test cases across 12 files (55 total, up from 11). Covers compliance failures (exit 1), usage errors (exit 4), and success regression guards (exit 0) for all assert, attest --assert, and evaluate command families.
1 parent 976a39e commit f8d0feb

12 files changed

Lines changed: 174 additions & 130 deletions

‎cmd/kosli/assertApproval_test.go‎

Lines changed: 44 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,11 @@ func (suite *AssertApprovalCommandTestSuite) SetupTest() {
4949
func (suite *AssertApprovalCommandTestSuite) TestAssertApprovalCmd() {
5050
tests := []cmdTestCase{
5151
{
52-
wantError: true,
53-
name: "1 missing --org fails",
54-
cmd: fmt.Sprintf(`assert approval --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName),
55-
golden: "Error: --org is not set\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
52+
wantError: true,
53+
wantExitCode: 4,
54+
name: "1 missing --org fails",
55+
cmd: fmt.Sprintf(`assert approval --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName),
56+
golden: "Error: --org is not set\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
5657
},
5758
{
5859
wantError: true,
@@ -61,21 +62,24 @@ func (suite *AssertApprovalCommandTestSuite) TestAssertApprovalCmd() {
6162
golden: "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-approval' belonging to organization 'docs-cmd-test-user'\n",
6263
},
6364
{
64-
wantError: true,
65-
name: "3 asserting an existing artifact that does not have an approval (using --fingerprint) works and exits with non-zero code",
66-
cmd: fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
67-
golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
65+
wantError: true,
66+
wantExitCode: 1,
67+
name: "3 asserting an existing artifact that does not have an approval (using --fingerprint) works and exits with non-zero code",
68+
cmd: fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
69+
golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
6870
},
6971
{
70-
wantError: true,
71-
name: "4 asserting approval of an existing artifact that does not have an approval (using --artifact-type) works and exits with non-zero code",
72-
cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
73-
golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
72+
wantError: true,
73+
wantExitCode: 1,
74+
name: "4 asserting approval of an existing artifact that does not have an approval (using --artifact-type) works and exits with non-zero code",
75+
cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
76+
golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
7477
},
7578
{
76-
name: "5 asserting approval of an existing artifact that has an approval (using --artifact-type) works and exits with zero code",
77-
cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
78-
golden: "artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is approved (approval no. [1])\n",
79+
name: "5 asserting approval of an existing artifact that has an approval (using --artifact-type) works and exits with zero code",
80+
wantExitCode: 0,
81+
cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
82+
golden: "artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is approved (approval no. [1])\n",
7983
additionalConfig: assertApprovalTestConfig{
8084
createApproval: true,
8185
isRequest: false,
@@ -88,39 +92,44 @@ func (suite *AssertApprovalCommandTestSuite) TestAssertApprovalCmd() {
8892
golden: "artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is approved (approval no. [1])\n",
8993
},
9094
{
91-
wantError: true,
92-
name: "7 not providing --fingerprint nor --artifact-type fails",
93-
cmd: fmt.Sprintf(`assert approval --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
94-
golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
95+
wantError: true,
96+
wantExitCode: 4,
97+
name: "7 not providing --fingerprint nor --artifact-type fails",
98+
cmd: fmt.Sprintf(`assert approval --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
99+
golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
95100
},
96101
{
97-
wantError: true,
98-
name: "8 providing both --fingerprint and --artifact-type fails",
99-
cmd: fmt.Sprintf(`assert approval --artifact-type file --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
100-
golden: "Error: only one of --fingerprint, --artifact-type is allowed\n",
102+
wantError: true,
103+
wantExitCode: 4,
104+
name: "8 providing both --fingerprint and --artifact-type fails",
105+
cmd: fmt.Sprintf(`assert approval --artifact-type file --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
106+
golden: "Error: only one of --fingerprint, --artifact-type is allowed\n",
101107
},
102108
{
103-
wantError: true,
104-
name: "9 missing --flow fails",
105-
cmd: fmt.Sprintf(`assert approval --fingerprint %s %s`, suite.fingerprint, suite.defaultKosliArguments),
106-
golden: "Error: required flag(s) \"flow\" not set\n",
109+
wantError: true,
110+
wantExitCode: 4,
111+
name: "9 missing --flow fails",
112+
cmd: fmt.Sprintf(`assert approval --fingerprint %s %s`, suite.fingerprint, suite.defaultKosliArguments),
113+
golden: "Error: required flag(s) \"flow\" not set\n",
107114
},
108115
{
109-
wantError: true,
110-
name: "10 asserting approval of an unapproved existing artifact (using --artifact-type) works and exits with non-zero code",
111-
cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
112-
golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
116+
wantError: true,
117+
wantExitCode: 1,
118+
name: "10 asserting approval of an unapproved existing artifact (using --artifact-type) works and exits with non-zero code",
119+
cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
120+
golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
113121
additionalConfig: assertApprovalTestConfig{
114122
createApproval: true,
115123
isRequest: true,
116124
},
117125
},
118126
// The approval request created in test 9 is valid here too
119127
{
120-
wantError: true,
121-
name: "11 asserting approval of an unapproved existing artifact (using --fingerprint) works and exits with non-zero code",
122-
cmd: fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
123-
golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
128+
wantError: true,
129+
wantExitCode: 1,
130+
name: "11 asserting approval of an unapproved existing artifact (using --fingerprint) works and exits with non-zero code",
131+
cmd: fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
132+
golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
124133
},
125134
}
126135

‎cmd/kosli/assertArtifact_test.go‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -83,10 +83,11 @@ func (suite *AssertArtifactCommandTestSuite) SetupTest() {
8383
func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
8484
tests := []cmdTestCase{
8585
{
86-
wantError: true,
87-
name: "01 missing --org fails",
88-
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName1),
89-
golden: "Error: --org is not set\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
86+
wantError: true,
87+
wantExitCode: 4,
88+
name: "01 missing --org fails",
89+
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName1),
90+
golden: "Error: --org is not set\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
9091
},
9192
{
9293
wantError: true,
@@ -95,9 +96,10 @@ func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
9596
golden: "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-artifact-one' belonging to organization 'docs-cmd-test-user'\n",
9697
},
9798
{
98-
name: "03 asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
99-
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
100-
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
99+
name: "03 asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
100+
wantExitCode: 0,
101+
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
102+
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
101103
},
102104
{
103105
name: "04 json output of asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
@@ -176,28 +178,32 @@ func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
176178
},
177179
},
178180
{
179-
wantError: true,
180-
name: "14 not providing --fingerprint nor --artifact-type fails",
181-
cmd: fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
182-
golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
181+
wantError: true,
182+
wantExitCode: 4,
183+
name: "14 not providing --fingerprint nor --artifact-type fails",
184+
cmd: fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
185+
golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
183186
},
184187
{
185-
wantError: true,
186-
name: "15 providing both --environment and --polices fails",
187-
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --environment %s --policy %s %s`, suite.fingerprint1, suite.envName, suite.policyName1, suite.defaultKosliArguments),
188-
golden: "Error: Cannot specify both 'environment_name' and 'policy_name' at the same time\n",
188+
wantError: true,
189+
wantExitCode: 4,
190+
name: "15 providing both --environment and --polices fails",
191+
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --environment %s --policy %s %s`, suite.fingerprint1, suite.envName, suite.policyName1, suite.defaultKosliArguments),
192+
golden: "Error: Cannot specify both 'environment_name' and 'policy_name' at the same time\n",
189193
},
190194
{
191-
wantError: true,
192-
name: "16 asserting a single existing non-compliant artifact (using --fingerprint) results in non-zero exit",
193-
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint3, suite.defaultKosliArguments),
194-
goldenRegex: "^Error: NON-COMPLIANT\n",
195+
wantError: true,
196+
wantExitCode: 1,
197+
name: "16 asserting a single existing non-compliant artifact (using --fingerprint) results in non-zero exit",
198+
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint3, suite.defaultKosliArguments),
199+
goldenRegex: "^Error: NON-COMPLIANT\n",
195200
},
196201
{
197-
wantError: true,
198-
name: "17 asserting a single existing non-compliant artifact (using --artifact-type) results in non-zero exit",
199-
cmd: fmt.Sprintf(`assert artifact %s --artifact-type file %s`, suite.artifact3Path, suite.defaultKosliArguments),
200-
goldenRegex: "^Error: NON-COMPLIANT\n",
202+
wantError: true,
203+
wantExitCode: 1,
204+
name: "17 asserting a single existing non-compliant artifact (using --artifact-type) results in non-zero exit",
205+
cmd: fmt.Sprintf(`assert artifact %s --artifact-type file %s`, suite.artifact3Path, suite.defaultKosliArguments),
206+
goldenRegex: "^Error: NON-COMPLIANT\n",
201207
},
202208
}
203209

‎cmd/kosli/assertPRAzure_test.go‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,25 @@ func (suite *AssertPRAzureCommandTestSuite) SetupTest() {
3030
func (suite *AssertPRAzureCommandTestSuite) TestAssertPRAzureCmd() {
3131
tests := []cmdTestCase{
3232
{
33-
name: "assert Azure PR evidence passes when commit has a PR in Azure",
34-
cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
33+
name: "assert Azure PR evidence passes when commit has a PR in Azure",
34+
wantExitCode: 0,
35+
cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
3536
--commit e6b38318747f1c225e6d2cdba1e88aa00fbcae29` + suite.defaultKosliArguments,
3637
golden: "found [1] pull request(s) in Azure DevOps for commit: e6b38318747f1c225e6d2cdba1e88aa00fbcae29\n",
3738
},
3839
{
39-
wantError: true,
40-
name: "assert Azure PR evidence fails when commit has no PRs in Azure",
41-
cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
40+
wantError: true,
41+
wantExitCode: 1,
42+
name: "assert Azure PR evidence fails when commit has no PRs in Azure",
43+
cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
4244
--commit 58d8aad96e0dcd11ada3dc6650d23909eed336ed` + suite.defaultKosliArguments,
4345
golden: "Error: assert failed: found no pull request(s) in Azure DevOps for commit: 58d8aad96e0dcd11ada3dc6650d23909eed336ed\n",
4446
},
4547
{
46-
wantError: true,
47-
name: "assert Azure PR evidence fails when commit does not exist",
48-
cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
48+
wantError: true,
49+
wantExitCode: 1,
50+
name: "assert Azure PR evidence fails when commit does not exist",
51+
cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
4952
--commit c4fa4c2ce6bef984abc93be9258a85f9137ff1c9` + suite.defaultKosliArguments,
5053
golden: "Error: assert failed: found no pull request(s) in Azure DevOps for commit: c4fa4c2ce6bef984abc93be9258a85f9137ff1c9\n",
5154
},

‎cmd/kosli/assertPRBitbucket_test.go‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,25 @@ func (suite *AssertPRBitbucketCommandTestSuite) SetupTest() {
3030
func (suite *AssertPRBitbucketCommandTestSuite) TestAssertPRBitbucketCmd() {
3131
tests := []cmdTestCase{
3232
{
33-
name: "assert Bitbucket PR evidence passes when commit has a PR in bitbucket",
34-
cmd: `assert pullrequest bitbucket --bitbucket-workspace kosli-dev --repository cli-test
33+
name: "assert Bitbucket PR evidence passes when commit has a PR in bitbucket",
34+
wantExitCode: 0,
35+
cmd: `assert pullrequest bitbucket --bitbucket-workspace kosli-dev --repository cli-test
3536
--commit fd54040fc90e7e83f7b152619bfa18917b72c34f` + suite.defaultKosliArguments,
3637
golden: "found [1] pull request(s) in Bitbucket for commit: fd54040fc90e7e83f7b152619bfa18917b72c34f\n",
3738
},
3839
{
39-
wantError: true,
40-
name: "assert Bitbucket PR evidence fails when both password and access token are provided",
41-
cmd: `assert pullrequest bitbucket --bitbucket-workspace kosli-dev --repository cli-test
40+
wantError: true,
41+
wantExitCode: 4,
42+
name: "assert Bitbucket PR evidence fails when both password and access token are provided",
43+
cmd: `assert pullrequest bitbucket --bitbucket-workspace kosli-dev --repository cli-test
4244
--commit fd54040fc90e7e83f7b152619bfa18917b72c34f --bitbucket-password xxxx` + suite.defaultKosliArguments,
4345
golden: "Error: only one of --bitbucket-password, --bitbucket-access-token is allowed\n",
4446
},
4547
{
46-
wantError: true,
47-
name: "assert Bitbucket PR evidence fails when commit has no PRs in bitbucket",
48-
cmd: `assert pullrequest bitbucket --bitbucket-workspace kosli-dev --repository cli-test
48+
wantError: true,
49+
wantExitCode: 1,
50+
name: "assert Bitbucket PR evidence fails when commit has no PRs in bitbucket",
51+
cmd: `assert pullrequest bitbucket --bitbucket-workspace kosli-dev --repository cli-test
4952
--commit 3dce097040987c4693d2e4be817474d9d0063c93` + suite.defaultKosliArguments,
5053
golden: "Error: assert failed: found no pull request(s) in Bitbucket for commit: 3dce097040987c4693d2e4be817474d9d0063c93\n",
5154
},

‎cmd/kosli/assertPRGithub_test.go‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,25 @@ func (suite *AssertPRGithubCommandTestSuite) SetupTest() {
3030
func (suite *AssertPRGithubCommandTestSuite) TestAssertPRGithubCmd() {
3131
tests := []cmdTestCase{
3232
{
33-
name: "assert Github PR evidence passes when commit has a PR in github",
33+
name: "assert Github PR evidence passes when commit has a PR in github",
34+
wantExitCode: 0,
3435
cmd: `assert pullrequest github --github-org kosli-dev --repository cli
3536
--commit ` + testHelpers.GithubCommitWithPR() + suite.defaultKosliArguments,
3637
golden: fmt.Sprintf("found [1] pull request(s) in Github for commit: %s\n", testHelpers.GithubCommitWithPR()),
3738
},
3839
{
39-
wantError: true,
40-
name: "assert Github PR evidence fails when commit has no PRs in github",
41-
cmd: `assert pullrequest github --github-org kosli-dev --repository cli
40+
wantError: true,
41+
wantExitCode: 1,
42+
name: "assert Github PR evidence fails when commit has no PRs in github",
43+
cmd: `assert pullrequest github --github-org kosli-dev --repository cli
4244
--commit 19aab7f063147614451c88969602a10afbabb43d` + suite.defaultKosliArguments,
4345
golden: "Error: assert failed: found no pull request(s) in Github for commit: 19aab7f063147614451c88969602a10afbabb43d\n",
4446
},
4547
{
46-
wantError: true,
47-
name: "assert Github PR evidence fails when commit does not exist",
48-
cmd: `assert pullrequest github --github-org kosli-dev --repository cli
48+
wantError: true,
49+
wantExitCode: 1,
50+
name: "assert Github PR evidence fails when commit does not exist",
51+
cmd: `assert pullrequest github --github-org kosli-dev --repository cli
4952
--commit 19aab7f063147614451c88969602a10afba123ab` + suite.defaultKosliArguments,
5053
golden: "Error: assert failed: found no pull request(s) in Github for commit: 19aab7f063147614451c88969602a10afba123ab\n",
5154
},

0 commit comments

Comments
 (0)