Skip to content

Commit d80d56c

Browse files
authored
Add invariant config coverage test (#5779)
## Why Ensure that all resource have invariant test coverage. Recent example where lack of coverage caused to release a bug in migrate rewrite #5775
1 parent 3a3700e commit d80d56c

1 file changed

Lines changed: 137 additions & 0 deletions

File tree

acceptance/invariant_test.go

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package acceptance_test
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"slices"
7+
"strings"
8+
"testing"
9+
10+
"github.com/databricks/cli/bundle/config"
11+
"github.com/databricks/cli/libs/dyn"
12+
"github.com/databricks/cli/libs/dyn/yamlloader"
13+
"github.com/stretchr/testify/assert"
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
const invariantConfigsDir = "bundle/invariant/configs"
18+
19+
// LackingInvariantTest lists keys from config.ResourcesTypes that knowingly lack
20+
// a covering config in invariantConfigsDir. Keys match the ResourcesTypes
21+
// form: "<group>" for the resource itself, "<group>.permissions" / "<group>.grants"
22+
// for permissions/grants coverage. Add a config and remove the entry to close a gap;
23+
// the test fails if an entry here is actually covered, so the list only shrinks.
24+
var LackingInvariantTest = map[string]bool{
25+
"quality_monitors": true,
26+
27+
"alerts.permissions": true,
28+
"apps.permissions": true,
29+
"clusters.permissions": true,
30+
"dashboards.permissions": true,
31+
"experiments.permissions": true,
32+
"pipelines.permissions": true,
33+
"postgres_projects.permissions": true,
34+
"sql_warehouses.permissions": true,
35+
36+
"catalogs.grants": true,
37+
"external_locations.grants": true,
38+
"registered_models.grants": true,
39+
"vector_search_indexes.grants": true,
40+
"volumes.grants": true,
41+
}
42+
43+
// TestInvariantConfigsCoverage ensures that the invariant test configs in
44+
// bundle/invariant/configs cover every bundle resource type, and that resource
45+
// types supporting permissions or grants have at least one config exercising them.
46+
//
47+
// config.ResourcesTypes is the source of truth: it maps each resource group
48+
// (e.g. "jobs") to its Go type and, where the resource struct has a Permissions
49+
// or Grants field, adds derived keys "<group>.permissions" and "<group>.grants".
50+
func TestInvariantConfigsCoverage(t *testing.T) {
51+
present, withPermissions, withGrants := scanInvariantConfigs(t)
52+
53+
keys := make([]string, 0, len(config.ResourcesTypes))
54+
for key := range config.ResourcesTypes {
55+
keys = append(keys, key)
56+
}
57+
slices.Sort(keys)
58+
59+
for _, key := range keys {
60+
var covered bool
61+
var hint string
62+
switch {
63+
case strings.HasSuffix(key, ".permissions"):
64+
group := strings.TrimSuffix(key, ".permissions")
65+
covered = withPermissions[group]
66+
hint = "attaches permissions to a " + group + " resource"
67+
case strings.HasSuffix(key, ".grants"):
68+
group := strings.TrimSuffix(key, ".grants")
69+
covered = withGrants[group]
70+
hint = "attaches grants to a " + group + " resource"
71+
default:
72+
covered = present[key]
73+
hint = "defines a " + key + " resource"
74+
}
75+
76+
if LackingInvariantTest[key] {
77+
assert.False(t, covered,
78+
"%q is covered by a config in %s; remove it from LackingInvariantTest", key, invariantConfigsDir)
79+
continue
80+
}
81+
assert.True(t, covered,
82+
"no config in %s %s; add one or allowlist %q", invariantConfigsDir, hint, key)
83+
}
84+
}
85+
86+
// scanInvariantConfigs parses every config in the invariant configs directory and
87+
// returns the set of resource groups present, the groups with at least one resource
88+
// carrying permissions, and the groups with at least one resource carrying grants.
89+
func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants map[string]bool) {
90+
present = map[string]bool{}
91+
withPermissions = map[string]bool{}
92+
withGrants = map[string]bool{}
93+
94+
entries, err := os.ReadDir(invariantConfigsDir)
95+
require.NoError(t, err)
96+
97+
for _, entry := range entries {
98+
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yml.tmpl") {
99+
continue
100+
}
101+
path := filepath.Join(invariantConfigsDir, entry.Name())
102+
contents, err := os.ReadFile(path)
103+
require.NoError(t, err)
104+
105+
v, err := yamlloader.LoadYAML(path, strings.NewReader(string(contents)))
106+
require.NoError(t, err, "failed to parse %s", path)
107+
108+
resources := v.Get("resources")
109+
if resources.Kind() != dyn.KindMap {
110+
// Some configs (e.g. PyDABs) declare resources outside of YAML.
111+
continue
112+
}
113+
114+
for _, group := range resources.MustMap().Pairs() {
115+
groupName := group.Key.MustString()
116+
present[groupName] = true
117+
118+
if group.Value.Kind() != dyn.KindMap {
119+
continue
120+
}
121+
for _, resource := range group.Value.MustMap().Pairs() {
122+
cfg := resource.Value
123+
if cfg.Kind() != dyn.KindMap {
124+
continue
125+
}
126+
if cfg.Get("permissions").Kind() != dyn.KindInvalid {
127+
withPermissions[groupName] = true
128+
}
129+
if cfg.Get("grants").Kind() != dyn.KindInvalid {
130+
withGrants[groupName] = true
131+
}
132+
}
133+
}
134+
}
135+
136+
return present, withPermissions, withGrants
137+
}

0 commit comments

Comments
 (0)