Skip to content

Commit 262c54e

Browse files
dangrondahldangrondahl
committed
green: Layer 2 exit code confidence — wantExitCode in cmdTestCase + scenario tests
- ExitCodeFor now recognises Cobra's built-in usage error patterns (unknown flag, required flag not set, wrong arg count) so tests that use executeCommandC get the same exit code as the real binary - Add wantExitCode int to cmdTestCase; runTestCmd asserts ExitCodeFor(err) when the field is non-zero - New TestExitCodeScenarios covers codes 0–4 end-to-end using httptest (no local server required): version (0), deny-all policy (1), 5xx / unreachable (2), 401/403 (3), unknown flag / cobra required flag / wrong arg count (4) - Add wantExitCode: 4 to all arg/flag usage error cases in evaluateTrail and wantExitCode: 1 to all deny-all policy cases - Add wantExitCode annotations to assertSnapshot tests 01 (exit 4) and 04 (exit 1) - Extend TestExitCodeFor with four Cobra error pattern cases
1 parent efcf325 commit 262c54e

6 files changed

Lines changed: 221 additions & 42 deletions

File tree

‎cmd/kosli/assertSnapshot_test.go‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,11 @@ func (suite *AssertSnapshotCommandTestSuite) SetupTest() {
6464
func (suite *AssertSnapshotCommandTestSuite) TestAssertSnapshotCmd() {
6565
tests := []cmdTestCase{
6666
{
67-
wantError: true,
68-
name: "01 missing --org fails",
69-
cmd: fmt.Sprintf(`assert snapshot %s --api-token secret`, suite.nonCompliantEnvName),
70-
golden: "Error: --org is not set\nUsage: kosli assert snapshot ENVIRONMENT-NAME-OR-EXPRESSION [flags]\n",
67+
wantError: true,
68+
wantExitCode: 4,
69+
name: "01 missing --org fails",
70+
cmd: fmt.Sprintf(`assert snapshot %s --api-token secret`, suite.nonCompliantEnvName),
71+
golden: "Error: --org is not set\nUsage: kosli assert snapshot ENVIRONMENT-NAME-OR-EXPRESSION [flags]\n",
7172
},
7273
{
7374
wantError: true,
@@ -82,9 +83,10 @@ func (suite *AssertSnapshotCommandTestSuite) TestAssertSnapshotCmd() {
8283
golden: "Error: Environment named 'non-existing' does not exist for organization 'docs-cmd-test-user'\n",
8384
},
8485
{
85-
wantError: true,
86-
name: "04 asserting a non compliant env results in INCOMPLIANT and non-zero exit",
87-
cmd: fmt.Sprintf(`assert snapshot %s %s`, suite.nonCompliantEnvName, suite.defaultKosliArguments),
86+
wantError: true,
87+
wantExitCode: 1,
88+
name: "04 asserting a non compliant env results in INCOMPLIANT and non-zero exit",
89+
cmd: fmt.Sprintf(`assert snapshot %s %s`, suite.nonCompliantEnvName, suite.defaultKosliArguments),
8890
additionalConfig: assertSnapshotTestConfig{
8991
reportToEnv: true,
9092
envName: suite.nonCompliantEnvName,

‎cmd/kosli/evaluateTrail_test.go‎

Lines changed: 43 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -37,33 +37,38 @@ func (suite *EvaluateTrailCommandTestSuite) SetupTest() {
3737
func (suite *EvaluateTrailCommandTestSuite) TestEvaluateTrailCmd() {
3838
tests := []cmdTestCase{
3939
{
40-
wantError: true,
41-
name: "missing trail name argument fails",
42-
cmd: fmt.Sprintf(`evaluate trail --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
43-
golden: "Error: accepts 1 arg(s), received 0\n",
40+
wantError: true,
41+
wantExitCode: 4,
42+
name: "missing trail name argument fails",
43+
cmd: fmt.Sprintf(`evaluate trail --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
44+
golden: "Error: accepts 1 arg(s), received 0\n",
4445
},
4546
{
46-
wantError: true,
47-
name: "providing more than one argument fails",
48-
cmd: fmt.Sprintf(`evaluate trail %s xxx --flow %s %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
49-
golden: "Error: accepts 1 arg(s), received 2\n",
47+
wantError: true,
48+
wantExitCode: 4,
49+
name: "providing more than one argument fails",
50+
cmd: fmt.Sprintf(`evaluate trail %s xxx --flow %s %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
51+
golden: "Error: accepts 1 arg(s), received 2\n",
5052
},
5153
{
52-
wantError: true,
53-
name: "missing --flow flag fails",
54-
cmd: fmt.Sprintf(`evaluate trail %s %s`, suite.trailName, suite.defaultKosliArguments),
55-
golden: "Error: required flag(s) \"flow\", \"policy\" not set\n",
54+
wantError: true,
55+
wantExitCode: 4,
56+
name: "missing --flow flag fails",
57+
cmd: fmt.Sprintf(`evaluate trail %s %s`, suite.trailName, suite.defaultKosliArguments),
58+
golden: "Error: required flag(s) \"flow\", \"policy\" not set\n",
5659
},
5760
{
58-
wantError: true,
59-
name: "missing --policy flag fails",
60-
cmd: fmt.Sprintf(`evaluate trail %s --flow %s %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
61-
golden: "Error: required flag(s) \"policy\" not set\n",
61+
wantError: true,
62+
wantExitCode: 4,
63+
name: "missing --policy flag fails",
64+
cmd: fmt.Sprintf(`evaluate trail %s --flow %s %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
65+
golden: "Error: required flag(s) \"policy\" not set\n",
6266
},
6367
{
64-
wantError: true,
65-
name: "missing --api-token fails",
66-
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/allow-all.rego --org orgX`, suite.trailName, suite.flowName),
68+
wantError: true,
69+
wantExitCode: 4,
70+
name: "missing --api-token fails",
71+
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/allow-all.rego --org orgX`, suite.trailName, suite.flowName),
6772
},
6873
{
6974
wantError: true,
@@ -76,9 +81,10 @@ func (suite *EvaluateTrailCommandTestSuite) TestEvaluateTrailCmd() {
7681
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/allow-all.rego %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
7782
},
7883
{
79-
wantError: true,
80-
name: "with --policy deny-all exits 1",
81-
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/deny-all.rego %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
84+
wantError: true,
85+
wantExitCode: 1,
86+
name: "with --policy deny-all exits 1",
87+
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/deny-all.rego %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
8288
},
8389
{
8490
wantError: true,
@@ -96,21 +102,23 @@ func (suite *EvaluateTrailCommandTestSuite) TestEvaluateTrailCmd() {
96102
goldenJson: []jsonCheck{{"allow", true}},
97103
},
98104
{
99-
wantError: true,
100-
name: "with --policy deny-all --output json prints JSON with allow false and violations",
101-
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/deny-all.rego --output json %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
102-
goldenRegex: `(?s)"allow":\s*false.*"violations":\s*\[.*"always denied"`,
105+
wantError: true,
106+
wantExitCode: 1,
107+
name: "with --policy deny-all --output json prints JSON with allow false and violations",
108+
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/deny-all.rego --output json %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
109+
goldenRegex: `(?s)"allow":\s*false.*"violations":\s*\[.*"always denied"`,
103110
},
104111
{
105112
name: "with --policy allow-all --output table prints allowed text",
106113
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/allow-all.rego --output table %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
107114
goldenRegex: `RESULT:\s+ALLOWED`,
108115
},
109116
{
110-
wantError: true,
111-
name: "with --policy deny-all --output table prints denied text with violations",
112-
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/deny-all.rego --output table %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
113-
goldenRegex: `RESULT:\s+DENIED\nVIOLATIONS:\s+always denied`,
117+
wantError: true,
118+
wantExitCode: 1,
119+
name: "with --policy deny-all --output table prints denied text with violations",
120+
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/deny-all.rego --output table %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
121+
goldenRegex: `RESULT:\s+DENIED\nVIOLATIONS:\s+always denied`,
114122
},
115123
{
116124
name: "with --policy allow-all and no --output defaults to table output",
@@ -129,10 +137,11 @@ func (suite *EvaluateTrailCommandTestSuite) TestEvaluateTrailCmd() {
129137
goldenJson: []jsonCheck{{"allow", true}, {"input.trail.name", suite.trailName}},
130138
},
131139
{
132-
wantError: true,
133-
name: "with --policy deny-all --output json --show-input includes input alongside allow and violations",
134-
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/deny-all.rego --output json --show-input %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
135-
goldenRegex: `(?s)"allow":\s*false.*"input":\s*\{.*"trail".*"violations":\s*\[.*"always denied"`,
140+
wantError: true,
141+
wantExitCode: 1,
142+
name: "with --policy deny-all --output json --show-input includes input alongside allow and violations",
143+
cmd: fmt.Sprintf(`evaluate trail %s --flow %s --policy testdata/policies/deny-all.rego --output json --show-input %s`, suite.trailName, suite.flowName, suite.defaultKosliArguments),
144+
goldenRegex: `(?s)"allow":\s*false.*"input":\s*\{.*"trail".*"violations":\s*\[.*"always denied"`,
136145
},
137146
{
138147
name: "with --policy allow-all --output table --show-input ignores show-input",

‎cmd/kosli/exitcodes_test.go‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"net/http/httptest"
7+
"testing"
8+
9+
kosliErrors "github.com/kosli-dev/cli/internal/errors"
10+
"github.com/stretchr/testify/assert"
11+
)
12+
13+
// TestExitCodeScenarios verifies that each documented exit code is produced
14+
// by the right class of error. Uses executeCommandC so no binary build is needed.
15+
//
16+
// Codes covered:
17+
//
18+
// 0 — success
19+
// 1 — compliance/policy violation
20+
// 2 — server unreachable or 5xx
21+
// 3 — invalid API token (401/403)
22+
// 4 — CLI usage error (missing flags, unknown flag, wrong arg count)
23+
func TestExitCodeScenarios(t *testing.T) {
24+
assertCode := func(t *testing.T, cmd string, want int) {
25+
t.Helper()
26+
_, _, err := executeCommandC(cmd)
27+
got := kosliErrors.ExitCodeFor(err)
28+
assert.Equal(t, want, got, "command: %s\nerror: %v", cmd, err)
29+
}
30+
31+
// ── exit 0: success ───────────────────────────────────────────────────────
32+
33+
t.Run("exit 0: kosli version succeeds", func(t *testing.T) {
34+
assertCode(t, "version", kosliErrors.ExitOK)
35+
})
36+
37+
// ── exit 1: compliance / policy violation ─────────────────────────────────
38+
39+
t.Run("exit 1: evaluate trail with deny-all policy", func(t *testing.T) {
40+
srv := newStaticJSONServer(t, http.StatusOK, `{"name":"my-trail","compliance_status":{}}`)
41+
cmd := fmt.Sprintf(
42+
"evaluate trail my-trail --flow my-flow --policy testdata/policies/deny-all.rego --host %s --org test-org --api-token secret",
43+
srv.URL,
44+
)
45+
assertCode(t, cmd, kosliErrors.ExitCompliance)
46+
})
47+
48+
// ── exit 2: server unreachable / 5xx ──────────────────────────────────────
49+
50+
t.Run("exit 2: server returns 500", func(t *testing.T) {
51+
srv := newStaticJSONServer(t, http.StatusInternalServerError, `{"message":"internal error"}`)
52+
cmd := fmt.Sprintf(
53+
"list environments --host %s --org test-org --api-token secret --max-api-retries 0",
54+
srv.URL,
55+
)
56+
assertCode(t, cmd, kosliErrors.ExitServer)
57+
})
58+
59+
t.Run("exit 2: server unreachable (bad host)", func(t *testing.T) {
60+
assertCode(t,
61+
"list environments --host http://localhost:19999 --org test-org --api-token secret --max-api-retries 0",
62+
kosliErrors.ExitServer,
63+
)
64+
})
65+
66+
// ── exit 3: invalid API token ─────────────────────────────────────────────
67+
68+
t.Run("exit 3: server returns 401", func(t *testing.T) {
69+
srv := newStaticJSONServer(t, http.StatusUnauthorized, `{"message":"unauthorized"}`)
70+
cmd := fmt.Sprintf(
71+
"list environments --host %s --org test-org --api-token bad-token",
72+
srv.URL,
73+
)
74+
assertCode(t, cmd, kosliErrors.ExitConfig)
75+
})
76+
77+
t.Run("exit 3: server returns 403", func(t *testing.T) {
78+
srv := newStaticJSONServer(t, http.StatusForbidden, `{"message":"forbidden"}`)
79+
cmd := fmt.Sprintf(
80+
"list environments --host %s --org test-org --api-token bad-token",
81+
srv.URL,
82+
)
83+
assertCode(t, cmd, kosliErrors.ExitConfig)
84+
})
85+
86+
// ── exit 4: CLI usage errors ──────────────────────────────────────────────
87+
88+
t.Run("exit 4: unknown flag", func(t *testing.T) {
89+
assertCode(t, "version --no-such-flag", kosliErrors.ExitUsage)
90+
})
91+
92+
t.Run("exit 4: cobra required flag not set (--flow, --policy)", func(t *testing.T) {
93+
assertCode(t,
94+
"evaluate trail my-trail --host http://localhost:8001 --org test-org --api-token secret",
95+
kosliErrors.ExitUsage,
96+
)
97+
})
98+
99+
t.Run("exit 4: wrong number of arguments", func(t *testing.T) {
100+
// evaluate trail requires exactly 1 positional argument
101+
assertCode(t,
102+
"evaluate trail --host http://localhost:8001 --org test-org --api-token secret",
103+
kosliErrors.ExitUsage,
104+
)
105+
})
106+
}
107+
108+
// newStaticJSONServer starts a test HTTP server that always responds with the
109+
// given status code and JSON body, and closes itself when the test ends.
110+
func newStaticJSONServer(t *testing.T, status int, body string) *httptest.Server {
111+
t.Helper()
112+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
113+
w.Header().Set("Content-Type", "application/json")
114+
w.WriteHeader(status)
115+
_, _ = fmt.Fprint(w, body)
116+
}))
117+
t.Cleanup(srv.Close)
118+
return srv
119+
}

‎cmd/kosli/testHelpers.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"testing"
1313

1414
"github.com/kosli-dev/cli/internal/gitview"
15+
kosliErrors "github.com/kosli-dev/cli/internal/errors"
1516
shellwords "github.com/mattn/go-shellwords"
1617
"github.com/pkg/errors"
1718
"github.com/spf13/cobra"
@@ -32,6 +33,7 @@ type cmdTestCase struct {
3233
goldenRegex string
3334
goldenJson []jsonCheck // Use like this for array {"[0].compliant", false}
3435
wantError bool
36+
wantExitCode int // when non-zero, asserts ExitCodeFor(err) == wantExitCode
3537
additionalConfig interface{}
3638
}
3739

@@ -80,6 +82,11 @@ func runTestCmd(t *testing.T, tests []cmdTestCase) {
8082
if (err != nil) != tt.wantError {
8183
t.Errorf("error expectation not matched\n\n WANT error is: %t\n\n but GOT: '%v'", tt.wantError, err)
8284
}
85+
if tt.wantExitCode != 0 {
86+
if got := kosliErrors.ExitCodeFor(err); got != tt.wantExitCode {
87+
t.Errorf("exit code mismatch: want %d, got %d (error: %v)", tt.wantExitCode, got, err)
88+
}
89+
}
8390
if tt.golden != "" {
8491
if !bytes.Equal([]byte(tt.golden), []byte(out)) {
8592
t.Errorf("does not match golden\n\nWANT:\n'%s'\n\nGOT:\n'%s'\n", tt.golden, out)

‎internal/errors/errors.go‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package errors
22

3-
import "errors"
3+
import (
4+
"errors"
5+
"strings"
6+
)
47

58
// Exit codes
69
const (
@@ -61,5 +64,23 @@ func ExitCodeFor(err error) int {
6164
if errors.As(err, &eu) {
6265
return ExitUsage
6366
}
67+
if isCobraUsageError(err) {
68+
return ExitUsage
69+
}
6470
return ExitCompliance // default non-zero; use exit 1 for unclassified errors
6571
}
72+
73+
// isCobraUsageError returns true for plain errors produced by Cobra's built-in
74+
// flag and argument validation that were not already wrapped as ErrUsage.
75+
// This covers cases where innerMain's wrapping is bypassed (e.g. in tests).
76+
func isCobraUsageError(err error) bool {
77+
if err == nil {
78+
return false
79+
}
80+
msg := err.Error()
81+
return strings.Contains(msg, "unknown flag:") ||
82+
strings.Contains(msg, "unknown shorthand flag:") ||
83+
strings.Contains(msg, "required flag(s)") ||
84+
strings.Contains(msg, "accepts") && strings.Contains(msg, "arg(s)") ||
85+
strings.Contains(msg, "requires at least") && strings.Contains(msg, "arg(s)")
86+
}

‎internal/errors/errors_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,25 @@ func TestExitCodeFor(t *testing.T) {
6161
err := fmt.Errorf("some unexpected error")
6262
assert.Equal(t, 1, kosliErrors.ExitCodeFor(err))
6363
})
64+
65+
// Cobra built-in errors are classified as exit 4 without explicit wrapping.
66+
t.Run("cobra unknown flag error returns 4", func(t *testing.T) {
67+
err := fmt.Errorf("unknown flag: --foo")
68+
assert.Equal(t, 4, kosliErrors.ExitCodeFor(err))
69+
})
70+
71+
t.Run("cobra required flag not set returns 4", func(t *testing.T) {
72+
err := fmt.Errorf(`required flag(s) "flow", "policy" not set`)
73+
assert.Equal(t, 4, kosliErrors.ExitCodeFor(err))
74+
})
75+
76+
t.Run("cobra ExactArgs error returns 4", func(t *testing.T) {
77+
err := fmt.Errorf("accepts 1 arg(s), received 0")
78+
assert.Equal(t, 4, kosliErrors.ExitCodeFor(err))
79+
})
80+
81+
t.Run("cobra MinimumNArgs error returns 4", func(t *testing.T) {
82+
err := fmt.Errorf("requires at least 1 arg(s), only received 0")
83+
assert.Equal(t, 4, kosliErrors.ExitCodeFor(err))
84+
})
6485
}

0 commit comments

Comments
 (0)