Skip to content

Commit dbf3606

Browse files
ToreMerkelyToreMerkely
authored
Assert artifact: Support that an artifact can exist in multiple flows server/#3953 (#559)
* Cleaned up assert tests so they work with new server image * Works with old server on staging * Have separate fingerprints for the single and multiple artifact in test * Fixed printing of environment * Added test for specifying policy and environment at the same time is failing * Changed order in table output
1 parent 9dcc708 commit dbf3606

5 files changed

Lines changed: 186 additions & 58 deletions

File tree

‎bin/get_file_from_test_integration.sh‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env bash
2+
3+
4+
SCRIPT_NAME=get_file_from_test_integration.sh
5+
FROM_FILE_PATH=""
6+
TO_FILE_NAME=""
7+
8+
die()
9+
{
10+
echo "Error: $1" >&2
11+
exit 1
12+
}
13+
14+
print_help()
15+
{
16+
cat <<EOF
17+
Usage: $SCRIPT_NAME [options] <DOCKER_FILE> [LOCAL_FILE]
18+
19+
Get a file from test_integration
20+
21+
Options are:
22+
-h Print this help menu
23+
EOF
24+
}
25+
26+
check_arguments()
27+
{
28+
while getopts "h" opt; do
29+
case $opt in
30+
h)
31+
print_help
32+
exit 1
33+
;;
34+
\?)
35+
echo "Invalid option: -$OPTARG" >&2
36+
exit 1
37+
;;
38+
esac
39+
done
40+
41+
# Remove options from command line
42+
shift $((OPTIND-1))
43+
44+
if [ $# -lt 1 ]; then
45+
echo "Error: Missing DOCKER_FILE argument"
46+
exit 1
47+
fi
48+
FROM_FILE_PATH=$1; shift
49+
if [ $# -eq 1 ]; then
50+
TO_FILE_NAME=$1; shift
51+
else
52+
TO_FILE_NAME=$(basename "${FROM_FILE_PATH}")
53+
fi
54+
}
55+
56+
main()
57+
{
58+
check_arguments "$@"
59+
docker exec cli_kosli_server sh -c "cat ${FROM_FILE_PATH}" > ${TO_FILE_NAME} || die "Failed to get file"
60+
}
61+
62+
main "$@"

‎cmd/kosli/assertArtifact.go‎

Lines changed: 31 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -149,36 +149,18 @@ func printAssertAsTable(raw string, out io.Writer, page int) error {
149149
if err != nil {
150150
return err
151151
}
152-
153-
flow, _ := evaluationResult["flow"].(string)
154-
trail, _ := evaluationResult["trail"].(string)
155152
scope := evaluationResult["scope"].(string)
156-
complianceStatus, _ := evaluationResult["compliance_status"].(map[string]interface{})
157-
attestationsStatuses, _ := complianceStatus["attestations_statuses"].([]interface{})
158153

159154
if evaluationResult["compliant"].(bool) {
160155
logger.Info("COMPLIANT")
161156
} else {
162157
logger.Info("Error: NON-COMPLIANT")
163158
}
164-
logger.Info("Flow: %v\nTrail %v", flow, trail)
165-
logger.Info("%-32v %-30v %-15v %-10v", "Attestation-name", "type", "status", "compliant")
166-
167-
for _, item := range attestationsStatuses {
168-
attestation := item.(map[string]interface{})
169-
name := attestation["attestation_name"]
170-
attType := attestation["attestation_type"]
171-
status := attestation["status"]
172-
isCompliant, _ := attestation["is_compliant"].(bool)
173-
unexpected, _ := attestation["unexpected"].(bool)
174-
unexpectedStr := ""
175-
if unexpected {
176-
unexpectedStr = "unexpected"
177-
}
178159

179-
logger.Info(" %-32v %-30v %-15v %-10v %-10v", name, attType, status, isCompliant, unexpectedStr)
180-
}
181160
if scope == "environment" || scope == "policy" {
161+
if scope == "environment" {
162+
logger.Info("Environment: %v", evaluationResult["environment"].(string))
163+
}
182164
logger.Info("%-32v %-30v", "Policy-name", "status")
183165
policyEvaluations := evaluationResult["policy_evaluations"].([]interface{})
184166
for _, item := range policyEvaluations {
@@ -220,8 +202,35 @@ func printAssertAsTable(raw string, out io.Writer, page int) error {
220202
}
221203
}
222204
}
205+
logger.Info("")
223206
}
224-
logger.Info("\nSee more details at %s", evaluationResult["html_url"].(string))
225207

208+
flows := evaluationResult["flows"].([]interface{})
209+
for _, item := range flows {
210+
item := item.(map[string]interface{})
211+
flow := item["flow"].(string)
212+
trail, _ := item["trail"].(string)
213+
complianceStatus, _ := item["compliance_status"].(map[string]interface{})
214+
attestationsStatuses, _ := complianceStatus["attestations_statuses"].([]interface{})
215+
216+
logger.Info("Flow: %v\n Trail: %v", flow, trail)
217+
logger.Info(" %-32v %-30v %-15v %-10v", "Attestation-name", "type", "status", "compliant")
218+
219+
for _, item := range attestationsStatuses {
220+
attestation := item.(map[string]interface{})
221+
name := attestation["attestation_name"]
222+
attType := attestation["attestation_type"]
223+
status := attestation["status"]
224+
isCompliant, _ := attestation["is_compliant"].(bool)
225+
unexpected, _ := attestation["unexpected"].(bool)
226+
unexpectedStr := ""
227+
if unexpected {
228+
unexpectedStr = "unexpected"
229+
}
230+
231+
logger.Info(" %-32v %-30v %-15v %-10v %-10v", name, attType, status, isCompliant, unexpectedStr)
232+
}
233+
logger.Info(" See more details at %s", evaluationResult["html_url"].(string))
234+
}
226235
return nil
227236
}

‎cmd/kosli/assertArtifact_test.go‎

Lines changed: 91 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -14,105 +14,160 @@ import (
1414
type AssertArtifactCommandTestSuite struct {
1515
suite.Suite
1616
defaultKosliArguments string
17-
flowName string
17+
flowName1 string
18+
flowName2 string
1819
envName string
1920
policyName1 string
2021
policyName2 string
21-
artifactName string
22-
artifactPath string
23-
fingerprint string
22+
artifactName1 string
23+
artifact1Path string
24+
fingerprint1 string
25+
artifactName2 string
26+
artifact2Path string
27+
fingerprint2 string
2428
}
2529

2630
func (suite *AssertArtifactCommandTestSuite) SetupTest() {
27-
suite.flowName = "assert-artifact"
31+
suite.flowName1 = "assert-artifact-one"
32+
suite.flowName2 = "assert-artifact-two"
2833
suite.envName = "assert-artifact-environment"
2934
suite.policyName1 = "assert-artifact-policy-1"
3035
suite.policyName2 = "assert-artifact-policy-2"
31-
suite.artifactName = "arti"
32-
suite.artifactPath = "testdata/folder1/hello.txt"
36+
suite.artifactName1 = "arti-for-AssertArtifactCommandTestSuite"
37+
suite.artifact1Path = "testdata/artifacts/AssertArtifactCommandTestSuiteArtifact1.txt"
38+
suite.artifactName2 = "arti-for-AssertArtifactCommandTestSuite2"
39+
suite.artifact2Path = "testdata/artifacts/AssertArtifactCommandTestSuiteArtifact2.txt"
3340
global = &GlobalOpts{
3441
ApiToken: "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpZCI6ImNkNzg4OTg5In0.e8i_lA_QrEhFncb05Xw6E_tkCHU9QfcY4OLTVUCHffY",
3542
Org: "docs-cmd-test-user",
3643
Host: "http://localhost:8001",
3744
}
3845
suite.defaultKosliArguments = fmt.Sprintf(" --host %s --org %s --api-token %s", global.Host, global.Org, global.ApiToken)
3946

40-
CreateFlow(suite.flowName, suite.Suite.T())
47+
CreateFlow(suite.flowName1, suite.Suite.T())
48+
CreateFlow(suite.flowName2, suite.Suite.T())
4149
fingerprintOptions := &fingerprintOptions{
4250
artifactType: "file",
4351
}
4452
CreateEnv(global.Org, suite.envName, "server", suite.Suite.T())
4553
CreatePolicy(global.Org, suite.policyName1, suite.Suite.T())
4654
CreatePolicy(global.Org, suite.policyName2, suite.Suite.T())
4755
var err error
48-
suite.fingerprint, err = GetSha256Digest(suite.artifactPath, fingerprintOptions, logger)
56+
suite.fingerprint1, err = GetSha256Digest(suite.artifact1Path, fingerprintOptions, logger)
4957
require.NoError(suite.Suite.T(), err)
50-
CreateArtifact(suite.flowName, suite.fingerprint, suite.artifactName, suite.Suite.T())
58+
CreateArtifact(suite.flowName1, suite.fingerprint1, suite.artifactName1, suite.Suite.T())
59+
suite.fingerprint2, err = GetSha256Digest(suite.artifact2Path, fingerprintOptions, logger)
60+
require.NoError(suite.Suite.T(), err)
61+
CreateArtifact(suite.flowName1, suite.fingerprint2, suite.artifactName2, suite.Suite.T())
62+
CreateArtifact(suite.flowName2, suite.fingerprint2, suite.artifactName1, suite.Suite.T())
5163
}
5264

5365
func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
5466
tests := []cmdTestCase{
5567
{
5668
wantError: true,
57-
name: "missing --org fails",
58-
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName),
69+
name: "01 missing --org fails",
70+
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName1),
5971
golden: "Error: --org is not set\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
6072
},
6173
{
6274
wantError: true,
63-
name: "asserting a non existing artifact fails",
64-
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
65-
golden: "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-artifact' belonging to organization 'docs-cmd-test-user'\n",
75+
name: "02 asserting a non existing artifact fails",
76+
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
77+
golden: "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-artifact-one' belonging to organization 'docs-cmd-test-user'\n",
78+
},
79+
{
80+
name: "03 asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
81+
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
82+
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
6683
},
6784
{
68-
name: "asserting an existing compliant artifact (using --fingerprint) results in OK and zero exit",
69-
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
70-
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
85+
name: "04 json output of asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
86+
cmd: fmt.Sprintf(`assert artifact --output json --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
87+
goldenJson: []jsonCheck{
88+
{"compliant", true},
89+
{"scope", "flow"},
90+
{"flows", "length:1"},
91+
},
92+
},
93+
{
94+
name: "05 asserting a single existing compliant artifact (using --fingerprint) for an environment results in OK and zero exit",
95+
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --environment %s %s`, suite.fingerprint1, suite.envName, suite.defaultKosliArguments),
96+
goldenRegex: "(?s)^COMPLIANT\n.*Policy-name.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
97+
},
98+
{
99+
name: "06 json output of asserting a single existing compliant artifact (using --fingerprint) for an environment results in OK and zero exit",
100+
cmd: fmt.Sprintf(`assert artifact --output json --fingerprint %s --environment %s %s`, suite.fingerprint1, suite.envName, suite.defaultKosliArguments),
101+
goldenJson: []jsonCheck{
102+
{"compliant", true},
103+
{"scope", "environment"},
104+
{"environment", suite.envName},
105+
{"flows", "length:1"},
106+
},
71107
},
72108
{
73-
name: "asserting an existing compliant artifact (using --fingerprint) for an environment results in OK and zero exit",
74-
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --flow %s --environment %s %s`, suite.fingerprint, suite.flowName, suite.envName, suite.defaultKosliArguments),
75-
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*Policy-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
109+
name: "07 asserting a single existing compliant artifact (using --fingerprint) for an policy results in OK and zero exit",
110+
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --policy %s %s`, suite.fingerprint1, suite.policyName1, suite.defaultKosliArguments),
111+
goldenRegex: "(?s)^COMPLIANT\n.*Policy-name.*assert-artifact-policy-1.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n.*",
76112
},
77113
{
78-
name: "asserting an existing compliant artifact (using --fingerprint) for an policy results in OK and zero exit",
79-
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --flow %s --policy %s %s`, suite.fingerprint, suite.flowName, suite.policyName1, suite.defaultKosliArguments),
80-
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*Policy-name.*assert-artifact-policy-1.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
114+
name: "08 json output of asserting a single existing compliant artifact (using --fingerprint) for an policy results in OK and zero exit\"",
115+
cmd: fmt.Sprintf(`assert artifact --output json --fingerprint %s --policy %s %s`, suite.fingerprint1, suite.policyName1, suite.defaultKosliArguments),
116+
goldenJson: []jsonCheck{
117+
{"compliant", true},
118+
{"scope", "policy"},
119+
{"flows", "length:1"},
120+
},
81121
},
82122
{
83-
name: "asserting an existing compliant artifact (using --artifact-type) results in OK and zero exit",
84-
cmd: fmt.Sprintf(`assert artifact %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
85-
goldenRegex: "(?s)^COMPLIANT\n.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02?(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
123+
name: "09 asserting a single existing compliant artifact (using --artifact-type) results in OK and zero exit",
124+
cmd: fmt.Sprintf(`assert artifact %s --artifact-type file %s`, suite.artifact1Path, suite.defaultKosliArguments),
125+
goldenRegex: "(?s)^COMPLIANT\n.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc?(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
86126
},
87127
{
88-
name: "json output of asserting an existing compliant artifact (using --artifact-type) results in OK and zero exit",
89-
cmd: fmt.Sprintf(`assert artifact %s --output json --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
128+
name: "10 json output of asserting a single existing compliant artifact (using --artifact-type) results in OK and zero exit",
129+
cmd: fmt.Sprintf(`assert artifact %s --output json --artifact-type file %s`, suite.artifact1Path, suite.defaultKosliArguments),
90130
goldenJson: []jsonCheck{
91131
{"compliant", true},
92132
{"scope", "flow"},
133+
{"flows", "length:1"},
93134
},
94135
},
95136
{
96-
name: "json output of asserting an existing compliant artifact (using --fingerprint) for an policy results in correct json",
97-
cmd: fmt.Sprintf(`assert artifact %s --output json --artifact-type file --flow %s --policy=%s --policy=%s %s`, suite.artifactPath, suite.flowName, suite.policyName1, suite.policyName2, suite.defaultKosliArguments),
137+
name: "11 json output of asserting a single existing compliant artifact (using --fingerprint) for a policy results in correct json",
138+
cmd: fmt.Sprintf(`assert artifact %s --output json --artifact-type file --policy=%s --policy=%s %s`, suite.artifact1Path, suite.policyName1, suite.policyName2, suite.defaultKosliArguments),
98139
goldenJson: []jsonCheck{
99140
{"compliant", true},
100141
{"scope", "policy"},
101142
{"policy_evaluations", "length:2"},
102143
{"policy_evaluations.[0].policy_name", suite.policyName1},
103144
{"policy_evaluations.[1].policy_name", suite.policyName2}},
104145
},
146+
{
147+
name: "12 asserting a multi existing compliant artifact (using --fingerprint) results in OK and zero exit",
148+
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint2, suite.defaultKosliArguments),
149+
goldenRegex: "(?s)^COMPLIANT\n.*Flow: assert-artifact-one.*Flow: assert-artifact-two.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/130fabe054d8d90b5d899833cfc769253a39b38854eb0c64214b68b276ef07e8(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
150+
},
151+
{
152+
name: "13 json output of asserting a multi existing compliant artifact (using --fingerprint) results in OK and zero exit",
153+
cmd: fmt.Sprintf(`assert artifact --output json --fingerprint %s %s`, suite.fingerprint2, suite.defaultKosliArguments),
154+
goldenJson: []jsonCheck{
155+
{"compliant", true},
156+
{"scope", "flow"},
157+
{"flows", "length:2"},
158+
},
159+
},
105160
{
106161
wantError: true,
107-
name: "not providing --fingerprint nor --artifact-type fails",
108-
cmd: fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
162+
name: "14 not providing --fingerprint nor --artifact-type fails",
163+
cmd: fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
109164
golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
110165
},
111166
{
112167
wantError: true,
113-
name: "not providing --flow when the artifact has multiple instances fails with a server error",
114-
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint, suite.defaultKosliArguments),
115-
golden: fmt.Sprintf("Error: Fingerprint '%s' matched multiple artifacts in org '%s'. Please narrow the search to one flow.\n", suite.fingerprint, global.Org),
168+
name: "15 providing both --environment and --polices fails",
169+
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --environment %s --policy %s %s`, suite.fingerprint1, suite.envName, suite.policyName1, suite.defaultKosliArguments),
170+
golden: "Error: Cannot specify both 'environment_name' and 'policy_name' at the same time\n",
116171
},
117172
}
118173

‎cmd/kosli/testdata/artifacts/AssertArtifactCommandTestSuiteArtifact1.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This shall only be used by the AssertArtifactCommandTestSuite

‎cmd/kosli/testdata/artifacts/AssertArtifactCommandTestSuiteArtifact2.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This shall only be used by the AssertArtifactCommandTestSuite and is the second artifact

0 commit comments

Comments
 (0)