Skip to content

Commit 8fd65fe

Browse files
graham-chainlinkgraham-chainlink
authored
[CLD-759]: feat(pipeline): enable strict yaml unmarshal (#510)
Originated from [this support request](https://chainlink-core.slack.com/archives/C09AJMWA98E/p1759973431200199?thread_ts=1759969075.884959&cid=C09AJMWA98E), when user has a typo of the field, the yaml will be unmashaled and that field will be ignored , this can cause errors in the pipeline when a certain expected field is not defined because the field was a typo. Enable strict unmarshaling and throws an error when there is an unknown field in the yaml input. JIRA: https://smartcontract-it.atlassian.net/browse/CLD-759 ### Example If user defined this input struct for their changeset ``` type GrahamInput struct { FirstName string LastName string } ``` however their input yaml has ``` environment: testnet domain: exemplar changesets: - deploy_link_token: payload: FirstName: Alice LastName: Smith Email: Alice@gmail.com ``` When pipeline is executed, it will now throw this error ``` Error: failed to unmarshal payload: json: unknown field "Email" ```
1 parent ac5d12c commit 8fd65fe

4 files changed

Lines changed: 219 additions & 6 deletions

File tree

‎.changeset/neat-lamps-repair.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"chainlink-deployments-framework": minor
3+
---
4+
5+
feat: enable strict yaml unmarshalling
6+
7+
When unmarshalling from yaml input for pipelines, if there is a field not defined in the struct, an error will be returned. This helps catch typos and misconfigurations early.

‎changeset/resolvers/resolver.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"reflect"
88
"runtime"
99
"sort"
10+
"strings"
1011
"sync"
1112
)
1213

@@ -140,14 +141,18 @@ func CallResolver[C any](resolver ConfigResolver, payload json.RawMessage) (C, e
140141
// If the function expects a pointer, create the underlying type and get a pointer to it
141142
elemType := inType.Elem()
142143
elemPtr := reflect.New(elemType)
143-
if err := json.Unmarshal(payload, elemPtr.Interface()); err != nil {
144+
decoder := json.NewDecoder(strings.NewReader(string(payload)))
145+
decoder.DisallowUnknownFields()
146+
if err := decoder.Decode(elemPtr.Interface()); err != nil {
144147
return zero, fmt.Errorf("unmarshal payload into %v: %w", inType, err)
145148
}
146149
arg = elemPtr
147150
} else {
148151
// If the function expects a value, create a pointer, unmarshal, then get the value
149152
inPtr := reflect.New(inType)
150-
if err := json.Unmarshal(payload, inPtr.Interface()); err != nil {
153+
decoder := json.NewDecoder(strings.NewReader(string(payload)))
154+
decoder.DisallowUnknownFields()
155+
if err := decoder.Decode(inPtr.Interface()); err != nil {
151156
return zero, fmt.Errorf("unmarshal payload into %v: %w", inType, err)
152157
}
153158
arg = inPtr.Elem()

‎engine/cld/changeset/common.go‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"fmt"
77
"os"
88
"reflect"
9+
"strings"
910

1011
fresolvers "github.com/smartcontractkit/chainlink-deployments-framework/changeset/resolvers"
1112
fdeployment "github.com/smartcontractkit/chainlink-deployments-framework/deployment"
@@ -104,8 +105,10 @@ func (f WrappedChangeSet[C]) WithJSON(_ C, inputStr string) ConfiguredChangeSet
104105
return config, errors.New("'payload' field is required and cannot be empty")
105106
}
106107

107-
if err := json.Unmarshal([]byte(inputObject.Payload), &config); err != nil {
108-
return config, fmt.Errorf("failed to unmarshal input: %w", err)
108+
payloadDecoder := json.NewDecoder(strings.NewReader(string(inputObject.Payload)))
109+
payloadDecoder.DisallowUnknownFields()
110+
if err := payloadDecoder.Decode(&config); err != nil {
111+
return config, fmt.Errorf("failed to unmarshal payload: %w", err)
109112
}
110113

111114
return config, nil
@@ -161,8 +164,10 @@ func (f WrappedChangeSet[C]) WithEnvInput(opts ...EnvInputOption[C]) ConfiguredC
161164
return config, errors.New("'payload' field is required and cannot be empty")
162165
}
163166

164-
if err := json.Unmarshal(inputObject.Payload, &config); err != nil {
165-
return config, fmt.Errorf("failed to unmarshal input: %w", err)
167+
payloadDecoder := json.NewDecoder(strings.NewReader(string(inputObject.Payload)))
168+
payloadDecoder.DisallowUnknownFields()
169+
if err := payloadDecoder.Decode(&config); err != nil {
170+
return config, fmt.Errorf("failed to unmarshal payload: %w", err)
166171
}
167172

168173
if options.inputModifier != nil {

‎engine/cld/changeset/common_test.go‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,70 @@ func TestChangesets_WithJSON_EmptyInput(t *testing.T) {
8888
require.Nil(t, configs.InputChainOverrides) // Chain overrides should be nil when input is missing
8989
}
9090

91+
func TestChangesets_WithJSON_StrictPayloadUnmarshaling(t *testing.T) {
92+
t.Parallel()
93+
94+
type TestConfig struct {
95+
Value string `json:"value"`
96+
Count int `json:"count"`
97+
}
98+
99+
tests := []struct {
100+
name string
101+
inputJSON string
102+
expectError bool
103+
errorMsg string
104+
}{
105+
{
106+
name: "valid payload with only known fields",
107+
inputJSON: `{"payload":{"value":"test","count":42}}`,
108+
expectError: false,
109+
},
110+
{
111+
name: "invalid payload with unknown field",
112+
inputJSON: `{"payload":{"value":"test","count":42,"unknownField":"value"}}`,
113+
expectError: true,
114+
errorMsg: "failed to unmarshal payload: json: unknown field \"unknownField\"",
115+
},
116+
{
117+
name: "invalid payload with multiple unknown fields",
118+
inputJSON: `{"payload":{"value":"test","count":42,"unknownField1":"value1","unknownField2":"value2"}}`,
119+
expectError: true,
120+
errorMsg: "failed to unmarshal payload: json: unknown field \"unknownField1\"",
121+
},
122+
{
123+
name: "invalid payload with nested unknown field",
124+
inputJSON: `{"payload":{"value":"test","count":42,"nested":{"unknown":"field"}}}`,
125+
expectError: true,
126+
errorMsg: "failed to unmarshal payload: json: unknown field \"nested\"",
127+
},
128+
}
129+
130+
for _, tt := range tests {
131+
t.Run(tt.name, func(t *testing.T) {
132+
t.Parallel()
133+
134+
cs := deployment.CreateChangeSet(
135+
func(e deployment.Environment, config TestConfig) (deployment.ChangesetOutput, error) {
136+
return deployment.ChangesetOutput{}, nil
137+
},
138+
func(e deployment.Environment, config TestConfig) error { return nil },
139+
)
140+
env := deployment.Environment{Logger: logger.Test(t)}
141+
configured := Configure(cs).WithJSON(TestConfig{}, tt.inputJSON)
142+
143+
_, err := configured.Apply(env)
144+
145+
if tt.expectError {
146+
require.Error(t, err, "Expected error for test case: %s", tt.name)
147+
require.ErrorContains(t, err, tt.errorMsg, "Error message should contain expected text")
148+
} else {
149+
require.NoError(t, err, "Expected no error for test case: %s", tt.name)
150+
}
151+
})
152+
}
153+
}
154+
91155
func TestChangesets_WithEnvInput(t *testing.T) {
92156
expectedConfig := "config from env"
93157
t.Setenv("DURABLE_PIPELINE_INPUT", `{"payload":"`+expectedConfig+`"}`)
@@ -500,6 +564,138 @@ func TestWithConfigResolver_ChainOverrides(t *testing.T) {
500564
assert.Equal(t, []uint64{10, 20, 30}, configs.InputChainOverrides)
501565
}
502566

567+
func TestWithConfigResolver_StrictPayloadUnmarshaling(t *testing.T) {
568+
type TestInput struct {
569+
Value string `json:"value"`
570+
Count int `json:"count"`
571+
}
572+
573+
type TestOutput struct {
574+
Result string
575+
}
576+
577+
resolver := func(input TestInput) (TestOutput, error) {
578+
return TestOutput{Result: fmt.Sprintf("%s_%d", input.Value, input.Count)}, nil
579+
}
580+
581+
tests := []struct {
582+
name string
583+
inputJSON string
584+
expectError bool
585+
errorMsg string
586+
}{
587+
{
588+
name: "valid payload with only known fields",
589+
inputJSON: `{"payload":{"value":"test","count":42}}`,
590+
expectError: false,
591+
},
592+
{
593+
name: "invalid payload with unknown field",
594+
inputJSON: `{"payload":{"value":"test","count":42,"unknownField":"value"}}`,
595+
expectError: true,
596+
errorMsg: "config resolver failed: unmarshal payload into changeset.TestInput: json: unknown field \"unknownField\"",
597+
},
598+
{
599+
name: "invalid payload with multiple unknown fields",
600+
inputJSON: `{"payload":{"value":"test","count":42,"unknownField1":"value1","unknownField2":"value2"}}`,
601+
expectError: true,
602+
errorMsg: "config resolver failed: unmarshal payload into changeset.TestInput: json: unknown field \"unknownField1\"",
603+
},
604+
{
605+
name: "invalid payload with nested unknown field",
606+
inputJSON: `{"payload":{"value":"test","count":42,"nested":{"unknown":"field"}}}`,
607+
expectError: true,
608+
errorMsg: "config resolver failed: unmarshal payload into changeset.TestInput: json: unknown field \"nested\"",
609+
},
610+
}
611+
612+
for _, tt := range tests {
613+
t.Run(tt.name, func(t *testing.T) {
614+
t.Setenv("DURABLE_PIPELINE_INPUT", tt.inputJSON)
615+
616+
cs := deployment.CreateChangeSet(
617+
func(e deployment.Environment, config TestOutput) (deployment.ChangesetOutput, error) {
618+
return deployment.ChangesetOutput{}, nil
619+
},
620+
func(e deployment.Environment, config TestOutput) error { return nil },
621+
)
622+
env := deployment.Environment{Logger: logger.Test(t)}
623+
configured := Configure(cs).WithConfigResolver(resolver)
624+
625+
_, err := configured.Apply(env)
626+
627+
if tt.expectError {
628+
require.Error(t, err, "Expected error for test case: %s", tt.name)
629+
require.ErrorContains(t, err, tt.errorMsg, "Error message should contain expected text")
630+
} else {
631+
require.NoError(t, err, "Expected no error for test case: %s", tt.name)
632+
}
633+
})
634+
}
635+
}
636+
637+
func TestWithEnvInput_StrictPayloadUnmarshaling(t *testing.T) {
638+
type TestConfig struct {
639+
Value string `json:"value"`
640+
Count int `json:"count"`
641+
}
642+
643+
tests := []struct {
644+
name string
645+
inputJSON string
646+
expectError bool
647+
errorMsg string
648+
}{
649+
{
650+
name: "valid payload with only known fields",
651+
inputJSON: `{"payload":{"value":"test","count":42}}`,
652+
expectError: false,
653+
},
654+
{
655+
name: "invalid payload with unknown field",
656+
inputJSON: `{"payload":{"value":"test","count":42,"unknownField":"value"}}`,
657+
expectError: true,
658+
errorMsg: "failed to unmarshal payload: json: unknown field \"unknownField\"",
659+
},
660+
{
661+
name: "invalid payload with multiple unknown fields",
662+
inputJSON: `{"payload":{"value":"test","count":42,"unknownField1":"value1","unknownField2":"value2"}}`,
663+
expectError: true,
664+
errorMsg: "failed to unmarshal payload: json: unknown field \"unknownField1\"",
665+
},
666+
{
667+
name: "invalid payload with nested unknown field",
668+
inputJSON: `{"payload":{"value":"test","count":42,"nested":{"unknown":"field"}}}`,
669+
expectError: true,
670+
errorMsg: "failed to unmarshal payload: json: unknown field \"nested\"",
671+
},
672+
}
673+
674+
for _, tt := range tests {
675+
t.Run(tt.name, func(t *testing.T) {
676+
t.Setenv("DURABLE_PIPELINE_INPUT", tt.inputJSON)
677+
678+
cs := deployment.CreateChangeSet(
679+
func(e deployment.Environment, config TestConfig) (deployment.ChangesetOutput, error) {
680+
return deployment.ChangesetOutput{}, nil
681+
},
682+
func(e deployment.Environment, config TestConfig) error { return nil },
683+
)
684+
env := deployment.Environment{Logger: logger.Test(t)}
685+
configured := Configure(cs).WithEnvInput()
686+
687+
_, err := configured.Apply(env)
688+
689+
if tt.expectError {
690+
require.Error(t, err, "Expected error for test case: %s", tt.name)
691+
require.ErrorContains(t, err, tt.errorMsg, "Error message should contain expected text")
692+
} else {
693+
require.NoError(t, err, "Expected no error for test case: %s", tt.name)
694+
}
695+
})
696+
}
697+
}
698+
503699
func TestConfigurations_ConfigResolverInfo(t *testing.T) {
504700
t.Parallel()
505701

0 commit comments

Comments
 (0)