Skip to content

Commit 1b42b51

Browse files
committed
feat: add policy-driven config merging and comma-separated match filters
Add structural deep-copy config merging with Replace, Shared, and Merger policies. Expand match filters to support comma-separated, trimmed, URL-decoded patterns while preserving exclusions. Add focused coverage for merge precedence, aliasing, custom mergers, and pattern handling.
1 parent 4d778a5 commit 1b42b51

6 files changed

Lines changed: 576 additions & 19 deletions

File tree

collections/slice.go

Lines changed: 35 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -128,30 +128,25 @@ func MatchItem(item string, patterns ...string) (matches, negated bool) {
128128
return true, false
129129
}
130130

131+
patterns = normalizeMatchPatterns(patterns)
132+
if len(patterns) == 0 {
133+
return false, false
134+
}
135+
131136
slices.SortFunc(patterns, sortPatterns)
132137

133138
//process negations first
134139
for _, p := range patterns {
135-
pattern, err := url.QueryUnescape(strings.TrimSpace(p))
136-
if err != nil {
137-
continue
138-
}
139-
140-
if strings.HasPrefix(pattern, "!") {
141-
if matchPattern(item, strings.TrimPrefix(pattern, "!")) {
140+
if strings.HasPrefix(p, "!") {
141+
if matchPattern(item, strings.TrimPrefix(p, "!")) {
142142
return false, true
143143
}
144144
}
145145

146146
}
147147

148148
// then normal filters
149-
for _, p := range patterns {
150-
pattern, err := url.QueryUnescape(strings.TrimSpace(p))
151-
if err != nil {
152-
continue
153-
}
154-
149+
for _, pattern := range patterns {
155150
if matchPattern(item, pattern) {
156151
return true, false
157152
}
@@ -177,14 +172,14 @@ func MatchItems(item string, patterns ...string) bool {
177172
return true
178173
}
179174

180-
slices.SortFunc(patterns, sortPatterns)
175+
patterns = normalizeMatchPatterns(patterns)
176+
if len(patterns) == 0 {
177+
return false
178+
}
181179

182-
for _, p := range patterns {
183-
pattern, err := url.QueryUnescape(strings.TrimSpace(p))
184-
if err != nil {
185-
continue
186-
}
180+
slices.SortFunc(patterns, sortPatterns)
187181

182+
for _, pattern := range patterns {
188183
if strings.HasPrefix(pattern, "!") {
189184
if matchPattern(item, strings.TrimPrefix(pattern, "!")) {
190185
return false
@@ -241,6 +236,26 @@ func matchPattern(item, pattern string) bool {
241236
return false
242237
}
243238

239+
func normalizeMatchPatterns(patterns []string) []string {
240+
var normalized []string
241+
for _, pattern := range patterns {
242+
parts := strings.Split(pattern, ",")
243+
for _, part := range parts {
244+
part = strings.TrimSpace(part)
245+
if part == "" && len(parts) > 1 {
246+
continue
247+
}
248+
249+
decoded, err := url.QueryUnescape(part)
250+
if err != nil {
251+
continue
252+
}
253+
normalized = append(normalized, decoded)
254+
}
255+
}
256+
return normalized
257+
}
258+
244259
// sortPatterns defines the priority for sorting:
245260
// exclusions ("!") have higher priority than other patterns.
246261
func sortPatterns(a, b string) int {
@@ -260,6 +275,7 @@ func sortPatterns(a, b string) int {
260275
}
261276

262277
func IsExclusionOnlyPatterns(patterns []string) bool {
278+
patterns = normalizeMatchPatterns(patterns)
263279
if len(patterns) == 0 {
264280
return false
265281
}

collections/slice_test.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,36 @@ var _ = Describe("MatchItems", func() {
4141
Entry("case insensitive suffix wildcard", "Apple", []string{"*PLE"}, true),
4242
Entry("case insensitive glob", "Apple", []string{"*PPL*"}, true),
4343
Entry("case insensitive exclusion", "Apple", []string{"!apple"}, false),
44+
Entry("comma-separated include after exclusion matches", "Item2", []string{"!Item,Item2"}, true),
45+
Entry("comma-separated exclusion wins", "Item", []string{"!Item,Item2"}, false),
46+
Entry("comma-separated wildcard exclusion wins", "Item99", []string{"!Item*,Other"}, false),
47+
Entry("comma-separated wildcard exclusion allows include", "Other", []string{"!Item*,Other"}, true),
48+
Entry("comma-separated values trim raw token whitespace", "Item2", []string{" !Item , Item2 "}, true),
49+
Entry("comma-separated malformed URL token is skipped", "Item2", []string{"!Item,%zz"}, true),
50+
Entry("URL encoded comma remains a literal pattern character", "A,B", []string{"A%2CB,C"}, true),
4451
)
4552
})
4653

54+
var _ = Describe("MatchItem", func() {
55+
It("expands comma-separated include and exclude patterns", func() {
56+
matches, negated := MatchItem("Item2", "!Item,Item2")
57+
Expect(matches).To(BeTrue())
58+
Expect(negated).To(BeFalse())
59+
60+
matches, negated = MatchItem("Item", "!Item,Item2")
61+
Expect(matches).To(BeFalse())
62+
Expect(negated).To(BeTrue())
63+
})
64+
})
65+
66+
var _ = Describe("IsExclusionOnlyPatterns", func() {
67+
It("expands comma-separated patterns before checking exclusions", func() {
68+
Expect(IsExclusionOnlyPatterns([]string{"!Item,!Item2"})).To(BeTrue())
69+
Expect(IsExclusionOnlyPatterns([]string{"!Item,Item2"})).To(BeFalse())
70+
Expect(IsExclusionOnlyPatterns([]string{"!Item,"})).To(BeTrue())
71+
})
72+
})
73+
4774
var _ = Describe("Append", func() {
4875
It("should append string slices", func() {
4976
slices := [][]any{

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/flanksource/commons
33
go 1.25.1
44

55
require (
6+
dario.cat/mergo v1.0.2
67
github.com/aws/aws-sdk-go-v2 v1.41.1
78
github.com/aws/aws-sdk-go-v2/config v1.32.9
89
github.com/aws/aws-sdk-go-v2/credentials v1.19.9

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
22
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
3+
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
4+
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
35
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
46
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
57
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=

merge/merge.go

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
// Package merge layers one configuration value over another, and deep-copies
2+
// one, from the struct definitions themselves.
3+
//
4+
// It exists to delete a category of code: hand-written `if o.X != "" { s.X = o.X }`
5+
// chains, hand-written emptiness tests, and hand-written field-by-field copies.
6+
// Each of those has to be updated whenever a field is added, and each is a place
7+
// a field can be silently forgotten. Structural merging cannot forget a field,
8+
// so adding one to a spec needs no merge code at all.
9+
//
10+
// The default behaviour is:
11+
//
12+
// - scalars and strings — the override wins when it is set (a zero value means
13+
// "unset", so an override can turn a bool on but never off);
14+
// - slices — replaced wholesale when the override's is non-empty, because a
15+
// list of tools means exactly those tools;
16+
// - maps — merged key-wise, because a map is a set of independent settings;
17+
// - structs, including those behind a pointer — merged field by field, so
18+
// setting one sub-field does not erase its siblings.
19+
//
20+
// Policy names the exceptions, including the types whose merge is a domain rule
21+
// rather than a structural one and which therefore merge themselves.
22+
package merge
23+
24+
import (
25+
"fmt"
26+
"reflect"
27+
28+
"dario.cat/mergo"
29+
)
30+
31+
// Policy declares the exceptions to structural merging for one family of types.
32+
// The zero Policy is valid and means "no exceptions".
33+
type Policy struct {
34+
// Replace lists types that are indivisible: when the override's value is set
35+
// it replaces the base's wholesale instead of being merged into. A pointer
36+
// type counts as set when it is non-nil — which is how an explicit
37+
// *float64(0) or *bool(false) survives a merge that would otherwise read the
38+
// pointed-at zero as "unset" and drop it.
39+
//
40+
// Name a type by its zero value: merge.Policy{Replace: []any{(*float64)(nil)}}.
41+
Replace []any
42+
43+
// Shared lists types that are referenced, not owned: Apply assigns them like
44+
// Replace, and Clone copies the reference instead of walking it. Registries,
45+
// catalogs and caches belong here — deep-copying one is both wasteful and
46+
// wrong, because consumers rely on pointer identity.
47+
Shared []any
48+
49+
// Merger lists types that merge themselves. Each must declare the method
50+
// `Merge(T) T` on the type named; Apply calls base.Merge(override) and takes
51+
// the result instead of walking the type. It is where a merge rule that is a
52+
// domain decision rather than a structural one lives: a list that accumulates
53+
// across configuration layers instead of being replaced, or two fields that
54+
// only mean anything when they move together.
55+
//
56+
// The method always runs — a nil slice or map destination is materialised
57+
// first, so "the base had none" reaches Merge as an empty value rather than
58+
// silently bypassing it. A pointer type therefore cannot be a Merger (a nil
59+
// pointer has nothing to call the method on), and naming a type without that
60+
// exact method panics rather than quietly reverting to structural merging.
61+
Merger []any
62+
}
63+
64+
// With returns the union of two policies, so a nested type's policy can be
65+
// composed into its container's rather than restated.
66+
func (p Policy) With(other Policy) Policy {
67+
return Policy{
68+
Replace: append(append([]any{}, p.Replace...), other.Replace...),
69+
Shared: append(append([]any{}, p.Shared...), other.Shared...),
70+
Merger: append(append([]any{}, p.Merger...), other.Merger...),
71+
}
72+
}
73+
74+
// Apply returns base with override's set fields layered on top, per the package
75+
// default and policy's exceptions. Neither argument is mutated, and the result
76+
// shares no mutable memory with either — a merged spec can be edited without
77+
// reaching back into the config it inherited from.
78+
//
79+
// T must be a struct, map or slice type; anything else is a caller error and
80+
// panics, as does a mergo failure, which cannot occur for a well-formed T since
81+
// both operands are the same concrete type by construction.
82+
func Apply[T any](base, override T, policy Policy) T {
83+
mergers := mergerSet(policy.Merger)
84+
out := clone(base, policy, mergers)
85+
src := clone(override, policy, mergers)
86+
opts := []func(*mergo.Config){mergo.WithOverride}
87+
if t := policy.transformer(mergers); t != nil {
88+
opts = append(opts, mergo.WithTransformers(t))
89+
}
90+
if err := mergo.Merge(&out, src, opts...); err != nil {
91+
panic(fmt.Sprintf("merge %T: %v", base, err))
92+
}
93+
return out
94+
}
95+
96+
// Clone returns a deep copy of v. Interfaces, functions and channels are copied
97+
// by reference — they carry behaviour rather than configuration — as are the
98+
// types named in policy.Shared. Unexported fields are copied as they stand.
99+
func Clone[T any](v T, policy Policy) T {
100+
return clone(v, policy, nil)
101+
}
102+
103+
// clone is Clone plus Apply's extra duty: materialise the nil slices and maps
104+
// whose type declares its own Merge, so mergo — which skips its transformers for
105+
// a nil destination — cannot bypass one.
106+
func clone[T any](v T, policy Policy, materialize map[reflect.Type]bool) T {
107+
shared := typeSet(policy.Shared)
108+
if len(shared) == 0 {
109+
shared = nil
110+
}
111+
return cloneValue(reflect.ValueOf(&v).Elem(), shared, materialize).Interface().(T)
112+
}
113+
114+
func cloneValue(v reflect.Value, shared, materialize map[reflect.Type]bool) reflect.Value {
115+
if !v.IsValid() || shared[v.Type()] {
116+
return v
117+
}
118+
switch v.Kind() {
119+
case reflect.Pointer:
120+
if v.IsNil() {
121+
return v
122+
}
123+
out := reflect.New(v.Type().Elem())
124+
out.Elem().Set(cloneValue(v.Elem(), shared, materialize))
125+
return out
126+
case reflect.Slice:
127+
if v.IsNil() {
128+
if !materialize[v.Type()] {
129+
return v
130+
}
131+
return reflect.MakeSlice(v.Type(), 0, 0)
132+
}
133+
out := reflect.MakeSlice(v.Type(), v.Len(), v.Len())
134+
for i := range v.Len() {
135+
out.Index(i).Set(cloneValue(v.Index(i), shared, materialize))
136+
}
137+
return out
138+
case reflect.Map:
139+
if v.IsNil() {
140+
if !materialize[v.Type()] {
141+
return v
142+
}
143+
return reflect.MakeMapWithSize(v.Type(), 0)
144+
}
145+
out := reflect.MakeMapWithSize(v.Type(), v.Len())
146+
for iter := v.MapRange(); iter.Next(); {
147+
out.SetMapIndex(cloneValue(iter.Key(), shared, materialize), cloneValue(iter.Value(), shared, materialize))
148+
}
149+
return out
150+
case reflect.Array:
151+
out := reflect.New(v.Type()).Elem()
152+
for i := range v.Len() {
153+
out.Index(i).Set(cloneValue(v.Index(i), shared, materialize))
154+
}
155+
return out
156+
case reflect.Struct:
157+
out := reflect.New(v.Type()).Elem()
158+
out.Set(v)
159+
for i := range v.NumField() {
160+
if v.Type().Field(i).IsExported() {
161+
out.Field(i).Set(cloneValue(v.Field(i), shared, materialize))
162+
}
163+
}
164+
return out
165+
default:
166+
return v
167+
}
168+
}
169+
170+
// transformer returns the mergo hook that short-circuits every policy type, or
171+
// nil when the policy declares none.
172+
func (p Policy) transformer(mergers map[reflect.Type]bool) mergo.Transformers {
173+
assign := typeSet(append(append([]any{}, p.Replace...), p.Shared...))
174+
if len(assign) == 0 && len(mergers) == 0 {
175+
return nil
176+
}
177+
return policyTransformer{assign: assign, mergers: mergers}
178+
}
179+
180+
type policyTransformer struct {
181+
assign map[reflect.Type]bool
182+
mergers map[reflect.Type]bool
183+
}
184+
185+
func (t policyTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
186+
switch {
187+
case t.mergers[typ]:
188+
return func(dst, src reflect.Value) error {
189+
if dst.CanSet() {
190+
dst.Set(dst.MethodByName("Merge").Call([]reflect.Value{src})[0])
191+
}
192+
return nil
193+
}
194+
case t.assign[typ]:
195+
return func(dst, src reflect.Value) error {
196+
if isSet(src) && dst.CanSet() {
197+
dst.Set(src)
198+
}
199+
return nil
200+
}
201+
default:
202+
return nil
203+
}
204+
}
205+
206+
// isSet reports whether an override value carries an instruction. A pointer is
207+
// set when it is non-nil, so a pointer to a zero scalar still wins; every other
208+
// kind is set when it is not the zero value.
209+
func isSet(v reflect.Value) bool {
210+
switch v.Kind() {
211+
case reflect.Pointer, reflect.Interface, reflect.Map, reflect.Slice, reflect.Func, reflect.Chan:
212+
return !v.IsNil()
213+
default:
214+
return !v.IsZero()
215+
}
216+
}
217+
218+
// mergerSet validates that every Merger type can actually merge itself. The
219+
// signature is checked here rather than at the merge, because a type whose method
220+
// was renamed or re-typed would otherwise fall back to structural merging — the
221+
// silent drift this package exists to remove.
222+
func mergerSet(values []any) map[reflect.Type]bool {
223+
set := typeSet(values)
224+
for typ := range set {
225+
if typ.Kind() == reflect.Pointer {
226+
panic(fmt.Sprintf("merge: policy Merger type %s must not be a pointer — a nil pointer has no value to merge into", typ))
227+
}
228+
method, ok := typ.MethodByName("Merge")
229+
if !ok || method.Type.NumIn() != 2 || method.Type.In(1) != typ ||
230+
method.Type.NumOut() != 1 || method.Type.Out(0) != typ {
231+
panic(fmt.Sprintf("merge: policy Merger type %s must declare a method Merge(%s) %s", typ, typ, typ))
232+
}
233+
}
234+
return set
235+
}
236+
237+
func typeSet(values []any) map[reflect.Type]bool {
238+
if len(values) == 0 {
239+
return nil
240+
}
241+
set := make(map[reflect.Type]bool, len(values))
242+
for _, value := range values {
243+
typ := reflect.TypeOf(value)
244+
if typ == nil {
245+
panic("merge: policy type must be named by a typed zero value, e.g. (*float64)(nil)")
246+
}
247+
set[typ] = true
248+
}
249+
return set
250+
}

0 commit comments

Comments
 (0)