Skip to content

Commit 8687eaa

Browse files
tookyclaudeclaude[bot]
authored
feat: add kosli evaluate input subcommand (#743)
- Adds `kosli evaluate input` — evaluate a local JSON file (or stdin) against a Rego policy, with no API dependency - Enables local policy development and testing without needing a running Kosli server - Input shape is opaque to the command — the policy defines what it expects Prompted by [this discussion](https://kosli-internal.slack.com/archives/C08R3H7TQPQ/p1774640547890899) where we realised `kosli evaluate trail` always hits the API, so there's no way to iterate on policies locally. Dan raised conftest as the alternative, but our tooling should support this natively — Rego is our language choice, and local testing should be turnkey. In response to Alex's [comparison of Rego vs pipeline controls](https://kosli-internal.slack.com/archives/C08R3H7TQPQ/p1774638408769049) — as more controls move into `kosli evaluate`, a fast local feedback loop becomes essential. --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>
1 parent e0e22b9 commit 8687eaa

13 files changed

Lines changed: 270 additions & 11 deletions

‎TODO.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,16 @@
3434
- [x] Slice 17: Align test method naming
3535
- [x] Slice 18: Fail evaluation when rehydration errors occur (instead of silently swallowing them)
3636
- [x] Slice 19: Add Long descriptions, Example blocks, and docs feedback (policy contract hint, snyk trail example)
37+
38+
## kosli evaluate input
39+
40+
- [x] Slice 1: `evaluate input --input-file` with a file path
41+
- [x] Slice 2: stdin support (omit --input-file to read stdin; `-` not supported by cobra)
42+
- [x] Slice 3: help text and examples
43+
- [x] Slice 4: PR review feedback
44+
- [x] Remove "using OPA" from all evaluate command long descriptions
45+
- [x] Add test cases for policy validation errors (missing package policy, missing allow rule, deny without violations)
46+
- [x] Update help text examples with fixture-capture workflow
47+
- [x] Refactor: use `cmd.InOrStdin()` for testable stdin
48+
- [x] Refactor: embed `commonEvaluateOptions` to remove flag duplication
49+
- [x] Slice 5: Detect terminal stdin and error when no input is piped

‎cmd/kosli/evaluate.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,16 @@ import (
66
"github.com/spf13/cobra"
77
)
88

9-
const evaluateShortDesc = `Evaluate Kosli trail data against OPA/Rego policies.`
9+
const evaluateShortDesc = `Evaluate data against Rego policies.`
1010

1111
// Backtick breaks (`"` + "`x`" + `"`) are needed to embed markdown
1212
// inline code spans inside raw string literals.
1313
const evaluateLongDesc = evaluateShortDesc + `
14-
Fetch trail data from Kosli and evaluate it against custom policies written
15-
in Rego, the policy language used by Open Policy Agent (OPA).
14+
Evaluate trail data or local JSON input against custom Rego policies.
15+
16+
Use ` + "`evaluate trail`" + ` or ` + "`evaluate trails`" + ` to fetch data from Kosli and evaluate it.
17+
Use ` + "`evaluate input`" + ` to evaluate a local JSON file or stdin without any API calls.
18+
1619
The policy must use ` + "`package policy`" + ` and define an ` + "`allow`" + ` rule.
1720
An optional ` + "`violations`" + ` rule (a set of strings) can provide human-readable denial reasons.
1821
The command exits with code 0 when allowed and code 1 when denied.`
@@ -28,6 +31,7 @@ func newEvaluateCmd(out io.Writer) *cobra.Command {
2831
cmd.AddCommand(
2932
newEvaluateTrailCmd(out),
3033
newEvaluateTrailsCmd(out),
34+
newEvaluateInputCmd(out),
3135
)
3236

3337
return cmd

‎cmd/kosli/evaluateHelpers.go‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,6 @@ func (o *commonEvaluateOptions) addFlags(cmd *cobra.Command, policyDesc string)
2828
cmd.Flags().StringVarP(&o.output, "output", "o", "table", outputFlag)
2929
cmd.Flags().BoolVar(&o.showInput, "show-input", false, "[optional] Include the policy input data in the output.")
3030
cmd.Flags().StringSliceVar(&o.attestations, "attestations", nil, "[optional] Limit which attestations are included. Plain name for trail-level, dot-qualified (artifact.name) for artifact-level.")
31-
32-
err := RequireFlags(cmd, []string{"flow", "policy"})
33-
if err != nil {
34-
logger.Error("failed to configure required flags: %v", err)
35-
}
3631
}
3732

3833
func fetchAndEnrichTrail(flowName, trailName string, attestations []string) (interface{}, error) {

‎cmd/kosli/evaluateInput.go‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"io"
7+
"os"
8+
9+
"github.com/spf13/cobra"
10+
"golang.org/x/term"
11+
)
12+
13+
type evaluateInputOptions struct {
14+
commonEvaluateOptions
15+
inputFile string
16+
}
17+
18+
const evaluateInputShortDesc = `Evaluate a local JSON input against a Rego policy.`
19+
20+
const evaluateInputLongDesc = evaluateInputShortDesc + `
21+
Read JSON from a file or stdin and evaluate it against a Rego policy.
22+
The input file should contain the raw JSON object your policy expects —
23+
not the wrapper produced by ` + "`--show-input`" + `. Use ` + "`jq '.input'`" + ` to extract
24+
the policy input from a ` + "`--show-input --output json`" + ` capture.
25+
26+
The policy must use ` + "`package policy`" + ` and define an ` + "`allow`" + ` rule.
27+
An optional ` + "`violations`" + ` rule (a set of strings) can provide human-readable denial reasons.
28+
The command exits with code 0 when allowed and code 1 when denied.
29+
30+
When ` + "`--input-file`" + ` is omitted, JSON is read from stdin.`
31+
32+
const evaluateInputExample = `
33+
# capture trail data for local policy iteration:
34+
kosli evaluate trail TRAIL --flow FLOW \
35+
--policy allow-all.rego \
36+
--show-input --output json | jq '.input' > trail-data.json
37+
38+
# then iterate on your policy locally:
39+
kosli evaluate input \
40+
--input-file trail-data.json \
41+
--policy policy.rego
42+
43+
# evaluate and show the data passed to the policy:
44+
kosli evaluate input \
45+
--input-file trail-data.json \
46+
--policy policy.rego \
47+
--show-input \
48+
--output json
49+
50+
# read input from stdin:
51+
cat trail-data.json | kosli evaluate input \
52+
--policy policy.rego`
53+
54+
func newEvaluateInputCmd(out io.Writer) *cobra.Command {
55+
o := new(evaluateInputOptions)
56+
cmd := &cobra.Command{
57+
Use: "input",
58+
Short: evaluateInputShortDesc,
59+
Long: evaluateInputLongDesc,
60+
Example: evaluateInputExample,
61+
Args: cobra.NoArgs,
62+
RunE: func(cmd *cobra.Command, args []string) error {
63+
return o.run(out, cmd.InOrStdin())
64+
},
65+
}
66+
67+
o.addFlags(cmd, "Path to a Rego policy file to evaluate against the input.")
68+
cmd.Flags().StringVarP(&o.inputFile, "input-file", "i", "", "[optional] Path to a JSON input file. Reads from stdin if omitted.")
69+
70+
cmd.Flags().Lookup("flow").Hidden = true
71+
cmd.Flags().Lookup("attestations").Hidden = true
72+
73+
err := RequireFlags(cmd, []string{"policy"})
74+
if err != nil {
75+
logger.Error("failed to configure required flags: %v", err)
76+
}
77+
78+
return cmd
79+
}
80+
81+
func (o *evaluateInputOptions) run(out io.Writer, in io.Reader) error {
82+
var input map[string]interface{}
83+
var err error
84+
85+
if o.inputFile == "" {
86+
if f, ok := in.(*os.File); ok && term.IsTerminal(int(f.Fd())) {
87+
return fmt.Errorf("no input provided: use --input-file or pipe JSON to stdin")
88+
}
89+
input, err = loadInput(in)
90+
} else {
91+
input, err = loadInputFromFile(o.inputFile)
92+
}
93+
if err != nil {
94+
return err
95+
}
96+
97+
return evaluateAndPrintResult(out, o.policyFile, input, o.output, o.showInput)
98+
}
99+
100+
func loadInputFromFile(filePath string) (result map[string]interface{}, err error) {
101+
f, err := os.Open(filePath)
102+
if err != nil {
103+
return nil, fmt.Errorf("failed to read input file: %w", err)
104+
}
105+
defer func() {
106+
if cerr := f.Close(); cerr != nil && err == nil {
107+
err = cerr
108+
}
109+
}()
110+
return loadInput(f)
111+
}
112+
113+
func loadInput(r io.Reader) (map[string]interface{}, error) {
114+
var input map[string]interface{}
115+
if err := json.NewDecoder(r).Decode(&input); err != nil {
116+
return nil, fmt.Errorf("failed to parse input: %w", err)
117+
}
118+
return input, nil
119+
}

‎cmd/kosli/evaluateInput_test.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package main
2+
3+
import (
4+
"strings"
5+
"testing"
6+
7+
"github.com/stretchr/testify/require"
8+
"github.com/stretchr/testify/suite"
9+
)
10+
11+
type EvaluateInputCommandTestSuite struct {
12+
suite.Suite
13+
}
14+
15+
func (suite *EvaluateInputCommandTestSuite) TestEvaluateInputCmd() {
16+
tests := []cmdTestCase{
17+
{
18+
wantError: true,
19+
name: "missing --policy flag fails",
20+
cmd: "evaluate input",
21+
golden: "Error: required flag(s) \"policy\" not set\n",
22+
},
23+
{
24+
name: "allow-all policy with input file returns ALLOWED",
25+
cmd: "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/allow-all.rego",
26+
goldenRegex: `RESULT:\s+ALLOWED`,
27+
},
28+
{
29+
wantError: true,
30+
name: "deny-all policy with input file returns DENIED",
31+
cmd: "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/deny-all.rego",
32+
goldenRegex: `RESULT:\s+DENIED`,
33+
},
34+
{
35+
wantError: true,
36+
name: "non-existent input file returns error",
37+
cmd: "evaluate input --input-file testdata/evaluate/no-such-file.json --policy testdata/policies/allow-all.rego",
38+
goldenRegex: `failed to read input file:`,
39+
},
40+
{
41+
wantError: true,
42+
name: "invalid JSON input file returns error",
43+
cmd: "evaluate input --input-file testdata/policies/allow-all.rego --policy testdata/policies/allow-all.rego",
44+
goldenRegex: `failed to parse input:`,
45+
},
46+
{
47+
wantError: true,
48+
name: "missing --input-file reads from stdin (empty stdin fails)",
49+
cmd: "evaluate input --policy testdata/policies/allow-all.rego",
50+
goldenRegex: `failed to parse input:`,
51+
},
52+
{
53+
name: "JSON output with allow-all policy",
54+
cmd: "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/allow-all.rego --output json",
55+
goldenJson: []jsonCheck{
56+
{"allow", true},
57+
},
58+
},
59+
{
60+
wantError: true,
61+
name: "policy with wrong package returns error",
62+
cmd: "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/no-package-policy.rego",
63+
goldenRegex: `policy package must be 'package policy', got 'foo'`,
64+
},
65+
{
66+
wantError: true,
67+
name: "policy missing allow rule returns error",
68+
cmd: "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/no-allow-rule.rego",
69+
goldenRegex: `policy must declare an 'allow' rule`,
70+
},
71+
{
72+
wantError: true,
73+
name: "deny without violations rule returns DENIED with no violation messages",
74+
cmd: "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/deny-no-violations.rego",
75+
goldenRegex: `RESULT:\s+DENIED`,
76+
},
77+
{
78+
name: "show-input includes input in JSON output",
79+
cmd: "evaluate input --input-file testdata/evaluate/trail-input.json --policy testdata/policies/allow-all.rego --output json --show-input",
80+
goldenJson: []jsonCheck{
81+
{"allow", true},
82+
{"input.trail.name", "test-trail"},
83+
},
84+
},
85+
}
86+
runTestCmd(suite.T(), tests)
87+
}
88+
89+
func TestLoadInput(t *testing.T) {
90+
reader := strings.NewReader(`{"trail": {"name": "from-reader"}}`)
91+
input, err := loadInput(reader)
92+
require.NoError(t, err)
93+
trail, ok := input["trail"].(map[string]interface{})
94+
require.True(t, ok)
95+
require.Equal(t, "from-reader", trail["name"])
96+
}
97+
98+
func TestLoadInputInvalidJSON(t *testing.T) {
99+
reader := strings.NewReader(`not json`)
100+
_, err := loadInput(reader)
101+
require.Error(t, err)
102+
require.Contains(t, err.Error(), "failed to parse input")
103+
}
104+
105+
func TestEvaluateInputCommandTestSuite(t *testing.T) {
106+
suite.Run(t, new(EvaluateInputCommandTestSuite))
107+
}

‎cmd/kosli/evaluateTrail.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
const evaluateTrailShortDesc = `Evaluate a trail against a policy.`
1010

1111
const evaluateTrailLongDesc = evaluateTrailShortDesc + `
12-
Fetch a single trail from Kosli and evaluate it against a Rego policy using OPA.
12+
Fetch a single trail from Kosli and evaluate it against a Rego policy.
1313
The trail data is passed to the policy as ` + "`input.trail`" + `.
1414
1515
Use ` + "`--attestations`" + ` to enrich the input with detailed attestation data
@@ -67,6 +67,11 @@ func newEvaluateTrailCmd(out io.Writer) *cobra.Command {
6767

6868
o.addFlags(cmd, "Path to a Rego policy file to evaluate against the trail.")
6969

70+
err := RequireFlags(cmd, []string{"flow", "policy"})
71+
if err != nil {
72+
logger.Error("failed to configure required flags: %v", err)
73+
}
74+
7075
return cmd
7176
}
7277

‎cmd/kosli/evaluateTrails.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
const evaluateTrailsShortDesc = `Evaluate multiple trails against a policy.`
1010

1111
const evaluateTrailsLongDesc = evaluateTrailsShortDesc + `
12-
Fetch multiple trails from Kosli and evaluate them together against a Rego policy using OPA.
12+
Fetch multiple trails from Kosli and evaluate them together against a Rego policy.
1313
The trail data is passed to the policy as ` + "`input.trails`" + ` (an array), unlike
1414
` + "`evaluate trail`" + ` which passes ` + "`input.trail`" + ` (a single object).
1515
@@ -68,6 +68,11 @@ func newEvaluateTrailsCmd(out io.Writer) *cobra.Command {
6868

6969
o.addFlags(cmd, "Path to a Rego policy file to evaluate against the trails.")
7070

71+
err := RequireFlags(cmd, []string{"flow", "policy"})
72+
if err != nil {
73+
logger.Error("failed to configure required flags: %v", err)
74+
}
75+
7176
return cmd
7277
}
7378

‎cmd/kosli/testHelpers.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ func executeCommandC(cmd string) (*cobra.Command, string, string, string, error)
6363
root.SilenceErrors = false
6464
root.SetOut(outWriter)
6565
root.SetErr(errWriter)
66+
root.SetIn(new(bytes.Buffer))
6667
root.SetArgs(args)
6768

6869
c, err := root.ExecuteC()

‎cmd/kosli/testdata/evaluate/trail-input.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"trail": {"name": "test-trail", "compliance_status": {}}}

‎cmd/kosli/testdata/policies/deny-no-violations.rego‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
package policy
2+
3+
allow = false

0 commit comments

Comments
 (0)