From f8befdad857e71a005f7ed4f00ba85275639911d Mon Sep 17 00:00:00 2001 From: Rens Rooimans Date: Tue, 16 Jun 2026 16:39:49 +0200 Subject: [PATCH 1/2] Allow key-based yaml anchoring --- engine/cld/pipeline/input/yaml.go | 61 ++++++++++++----- engine/cld/pipeline/input/yaml_test.go | 92 ++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 16 deletions(-) diff --git a/engine/cld/pipeline/input/yaml.go b/engine/cld/pipeline/input/yaml.go index d80172146..3e2d79d90 100644 --- a/engine/cld/pipeline/input/yaml.go +++ b/engine/cld/pipeline/input/yaml.go @@ -128,27 +128,41 @@ func GetAllChangesetsInOrder(changesets any) ([]ChangesetItem, error) { return result, nil } +// anyToJSONMapKey stringifies a map key for JSON-compatible maps. +func anyToJSONMapKey(v any) string { + switch k := v.(type) { + case string: + return k + case json.Number: + return k.String() + case int: + return strconv.Itoa(k) + case int64: + return strconv.FormatInt(k, 10) + case uint64: + return strconv.FormatUint(k, 10) + case float64: + if k == math.Trunc(k) && (k >= 1e15 || k <= -1e15) { + return strconv.FormatFloat(k, 'f', 0, 64) + } + + return strconv.FormatFloat(k, 'f', -1, 64) + default: + return fmt.Sprintf("%v", k) + } +} + +func isYAMLMergeKey(node *yaml.Node) bool { + return node != nil && node.Kind == yaml.ScalarNode && (node.Value == "<<" || node.Tag == "!!merge") +} + // ConvertToJSONSafe recursively converts map[interface{}]interface{} to map[string]any. func ConvertToJSONSafe(data any) (any, error) { switch v := data.(type) { case map[interface{}]interface{}: result := make(map[string]any) for key, value := range v { - var keyStr string - switch k := key.(type) { - case string: - keyStr = k - case int: - keyStr = strconv.Itoa(k) - case int64: - keyStr = strconv.FormatInt(k, 10) - case uint64: - keyStr = strconv.FormatUint(k, 10) - case float64: - keyStr = strconv.FormatFloat(k, 'f', -1, 64) - default: - keyStr = fmt.Sprintf("%v", k) - } + keyStr := anyToJSONMapKey(key) convertedValue, err := ConvertToJSONSafe(value) if err != nil { @@ -345,7 +359,22 @@ func YamlNodeToAny(node *yaml.Node) any { for i := 0; i+1 < len(node.Content); i += 2 { key := node.Content[i] value := node.Content[i+1] - out[key.Value] = YamlNodeToAny(value) + if isYAMLMergeKey(key) { + mergeMap, ok := YamlNodeToAny(value).(map[string]any) + if !ok { + continue + } + for mk, mv := range mergeMap { + if _, exists := out[mk]; !exists { + out[mk] = mv + } + } + + continue + } + + keyStr := anyToJSONMapKey(YamlNodeToAny(key)) + out[keyStr] = YamlNodeToAny(value) } return out diff --git a/engine/cld/pipeline/input/yaml_test.go b/engine/cld/pipeline/input/yaml_test.go index b0a35d437..97a502c7e 100644 --- a/engine/cld/pipeline/input/yaml_test.go +++ b/engine/cld/pipeline/input/yaml_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/domain" ) @@ -185,6 +186,97 @@ func TestYamlNodeToAny_Nil(t *testing.T) { require.Nil(t, got) } +func TestYamlNodeToAny_AliasMapKeys(t *testing.T) { + t.Parallel() + + yamlContent := ` +sel: &sel_sepolia 16015286601757825753 +other: &sel_arb 3478487238524512106 +chains: + *sel_sepolia: &cfg + qualifier: UltraFastCurse + timelockMinDelay: 0 + *sel_arb: *cfg +` + var root yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) + + doc := YamlNodeToAny(&root).(map[string]any) + chains := doc["chains"].(map[string]any) + + require.Equal(t, map[string]any{ + "16015286601757825753": map[string]any{ + "qualifier": "UltraFastCurse", + "timelockMinDelay": json.Number("0"), + }, + "3478487238524512106": map[string]any{ + "qualifier": "UltraFastCurse", + "timelockMinDelay": json.Number("0"), + }, + }, chains) +} + +func TestYamlNodeToAny_MergeKeys(t *testing.T) { + t.Parallel() + + yamlContent := ` +base: &base_cfg + qualifier: UltraFastCurse + timelockMinDelay: 0 +chains: + 16015286601757825753: + <<: *base_cfg + proposer: + quorum: 1 +` + var root yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) + + doc := YamlNodeToAny(&root).(map[string]any) + chains := doc["chains"].(map[string]any) + chain := chains["16015286601757825753"].(map[string]any) + + require.Equal(t, "UltraFastCurse", chain["qualifier"]) + require.Equal(t, json.Number("0"), chain["timelockMinDelay"]) + require.Equal(t, map[string]any{"quorum": json.Number("1")}, chain["proposer"]) +} + +func TestBuildChangesetInputJSON_AliasChainSelectorKeys(t *testing.T) { + t.Parallel() + + yamlContent := ` +environment: staging_testnet +domain: ccv +changesets: + - deploy_mcms: + chainOverrides: + - &sel_sepolia 16015286601757825753 + payload: + adapterVersion: "1.0.0" + chains: + *sel_sepolia: + qualifier: UltraFastCurse + timelockMinDelay: 0 +` + dpYAML, err := ParseYAMLBytes([]byte(yamlContent)) + require.NoError(t, err) + + changesets, err := GetAllChangesetsInOrder(dpYAML.Changesets) + require.NoError(t, err) + require.Len(t, changesets, 1) + + inputJSON, err := BuildChangesetInputJSON(changesets[0].Name, changesets[0].Data) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal([]byte(inputJSON), &decoded)) + + payload := decoded["payload"].(map[string]any) + chains := payload["chains"].(map[string]any) + require.Contains(t, chains, "16015286601757825753") + require.NotContains(t, chains, "sel_sepolia") +} + func TestSetChangesetEnvironmentVariable(t *testing.T) { t.Parallel() From 81945d9b638597904630ff561fd4924c828cd576 Mon Sep 17 00:00:00 2001 From: Graham Goh Date: Wed, 17 Jun 2026 14:45:57 +1000 Subject: [PATCH 2/2] fix(cld): harden YAML merge key parsing for pipeline inputs Return errors for invalid <<: targets, support sequence merges, and propagate YamlNodeToAny decode failures through the parse path. --- .changeset/loud-trams-jog.md | 5 + engine/cld/pipeline/input/resolve.go | 6 +- engine/cld/pipeline/input/yaml.go | 113 ++++++++++++---- engine/cld/pipeline/input/yaml_test.go | 175 ++++++++++++++++++++++++- 4 files changed, 266 insertions(+), 33 deletions(-) create mode 100644 .changeset/loud-trams-jog.md diff --git a/.changeset/loud-trams-jog.md b/.changeset/loud-trams-jog.md new file mode 100644 index 000000000..c8bbeba5c --- /dev/null +++ b/.changeset/loud-trams-jog.md @@ -0,0 +1,5 @@ +--- +"chainlink-deployments-framework": minor +--- + +feat(cld): enhance support for YAML anchors for pipeline input files diff --git a/engine/cld/pipeline/input/resolve.go b/engine/cld/pipeline/input/resolve.go index 958be883f..bd7ad440b 100644 --- a/engine/cld/pipeline/input/resolve.go +++ b/engine/cld/pipeline/input/resolve.go @@ -13,7 +13,11 @@ import ( // ResolveChangesetConfig resolves the configuration for a changeset using either // a registered resolver or keeping the original payload. func ResolveChangesetConfig(valueNode *yaml.Node, csName string, resolver resolvers.ConfigResolver) (any, error) { - changesetMap, ok := YamlNodeToAny(valueNode).(map[string]any) + changesetAny, err := yamlNodeToAny(valueNode) + if err != nil { + return nil, fmt.Errorf("decode changeset data for %s: %w", csName, err) + } + changesetMap, ok := changesetAny.(map[string]any) if !ok { return nil, fmt.Errorf("decode changeset data for %s: expected mapping node", csName) } diff --git a/engine/cld/pipeline/input/yaml.go b/engine/cld/pipeline/input/yaml.go index 3e2d79d90..135b0b537 100644 --- a/engine/cld/pipeline/input/yaml.go +++ b/engine/cld/pipeline/input/yaml.go @@ -153,7 +153,38 @@ func anyToJSONMapKey(v any) string { } func isYAMLMergeKey(node *yaml.Node) bool { - return node != nil && node.Kind == yaml.ScalarNode && (node.Value == "<<" || node.Tag == "!!merge") + return node != nil && + node.Kind == yaml.ScalarNode && + (node.Tag == "!!merge" || (node.Value == "<<" && node.Style == 0)) +} + +func mergeMapInto(dst, src map[string]any) { + for mk, mv := range src { + if _, exists := dst[mk]; !exists { + dst[mk] = mv + } + } +} + +func applyYAMLMerge(out map[string]any, value any) error { + switch v := value.(type) { + case map[string]any: + mergeMapInto(out, v) + + return nil + case []any: + for i, item := range v { + mergeMap, ok := item.(map[string]any) + if !ok { + return fmt.Errorf("YAML merge sequence entry %d must be a mapping, got %T", i, item) + } + mergeMapInto(out, mergeMap) + } + + return nil + default: + return fmt.Errorf("YAML merge key (<<) value must be a mapping or sequence of mappings, got %T", value) + } } // ConvertToJSONSafe recursively converts map[interface{}]interface{} to map[string]any. @@ -311,7 +342,11 @@ func ParseYAMLBytes(yamlData []byte) (*DurablePipelineYAML, error) { if err := yaml.Unmarshal(yamlData, &root); err != nil { return nil, fmt.Errorf("failed to parse YAML bytes: %w", err) } - rootMap, ok := YamlNodeToAny(&root).(map[string]any) + rootAny, err := yamlNodeToAny(&root) + if err != nil { + return nil, fmt.Errorf("failed to decode YAML: %w", err) + } + rootMap, ok := rootAny.(map[string]any) if !ok { return nil, errors.New("expected a YAML object at the root") } @@ -342,81 +377,103 @@ func ParseYAMLBytes(yamlData []byte) (*DurablePipelineYAML, error) { } // YamlNodeToAny converts a yaml.Node to a generic any value. +// This is the stable exported API; decode errors are ignored and nil is returned. func YamlNodeToAny(node *yaml.Node) any { - if node == nil { + v, err := yamlNodeToAny(node) + if err != nil { return nil } + return v +} + +// yamlNodeToAny converts a yaml.Node to a generic any value. +func yamlNodeToAny(node *yaml.Node) (any, error) { + if node == nil { + return nil, nil //nolint:nilnil // YAML null + } + switch node.Kind { case yaml.DocumentNode: if len(node.Content) == 0 { - return nil + return nil, nil //nolint:nilnil // empty YAML document } - return YamlNodeToAny(node.Content[0]) + return yamlNodeToAny(node.Content[0]) case yaml.MappingNode: out := make(map[string]any, len(node.Content)/2) for i := 0; i+1 < len(node.Content); i += 2 { key := node.Content[i] value := node.Content[i+1] if isYAMLMergeKey(key) { - mergeMap, ok := YamlNodeToAny(value).(map[string]any) - if !ok { - continue + merged, err := yamlNodeToAny(value) + if err != nil { + return nil, err } - for mk, mv := range mergeMap { - if _, exists := out[mk]; !exists { - out[mk] = mv - } + if err := applyYAMLMerge(out, merged); err != nil { + return nil, err } continue } - keyStr := anyToJSONMapKey(YamlNodeToAny(key)) - out[keyStr] = YamlNodeToAny(value) + keyAny, err := yamlNodeToAny(key) + if err != nil { + return nil, err + } + valueAny, err := yamlNodeToAny(value) + if err != nil { + return nil, err + } + keyStr := anyToJSONMapKey(keyAny) + out[keyStr] = valueAny } - return out + return out, nil case yaml.SequenceNode: out := make([]any, 0, len(node.Content)) for _, elem := range node.Content { - out = append(out, YamlNodeToAny(elem)) + elemAny, err := yamlNodeToAny(elem) + if err != nil { + return nil, err + } + out = append(out, elemAny) } - return out + return out, nil case yaml.ScalarNode: if node.Style == 0 && decimalInteger.MatchString(node.Value) { - return json.Number(node.Value) + return json.Number(node.Value), nil } switch node.Tag { case "!!int": if decimalInteger.MatchString(node.Value) { - return json.Number(node.Value) + return json.Number(node.Value), nil } if n, ok := new(big.Int).SetString(strings.ReplaceAll(node.Value, "_", ""), 0); ok { - return json.Number(n.String()) + return json.Number(n.String()), nil } - return node.Value + return node.Value, nil case "!!float": f, err := strconv.ParseFloat(node.Value, 64) if err != nil { - return node.Value + //nolint:nilerr // Fall back to raw scalar when float parsing fails. + return node.Value, nil } - return f + return f, nil case "!!null": - return nil + return nil, nil //nolint:nilnil // YAML null scalar case "!!bool": - return strings.EqualFold(node.Value, "true") + return strings.EqualFold(node.Value, "true"), nil default: - return node.Value + return node.Value, nil } case yaml.AliasNode: - return YamlNodeToAny(node.Alias) + return yamlNodeToAny(node.Alias) default: - return nil + return nil, nil //nolint:nilnil // unsupported node kind } } diff --git a/engine/cld/pipeline/input/yaml_test.go b/engine/cld/pipeline/input/yaml_test.go index 97a502c7e..cc46fe759 100644 --- a/engine/cld/pipeline/input/yaml_test.go +++ b/engine/cld/pipeline/input/yaml_test.go @@ -166,6 +166,11 @@ func TestConvertToJSONSafe(t *testing.T) { in: 42, want: 42, }, + { + name: "map with json number key", + in: map[interface{}]interface{}{json.Number("16015286601757825753"): "chain"}, + want: map[string]any{"16015286601757825753": "chain"}, + }, } for _, tt := range tests { @@ -182,8 +187,7 @@ func TestConvertToJSONSafe(t *testing.T) { func TestYamlNodeToAny_Nil(t *testing.T) { t.Parallel() - got := YamlNodeToAny(nil) - require.Nil(t, got) + require.Nil(t, YamlNodeToAny(nil)) } func TestYamlNodeToAny_AliasMapKeys(t *testing.T) { @@ -201,7 +205,9 @@ chains: var root yaml.Node require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) - doc := YamlNodeToAny(&root).(map[string]any) + docAny, err := yamlNodeToAny(&root) + require.NoError(t, err) + doc := docAny.(map[string]any) chains := doc["chains"].(map[string]any) require.Equal(t, map[string]any{ @@ -232,7 +238,9 @@ chains: var root yaml.Node require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) - doc := YamlNodeToAny(&root).(map[string]any) + docAny, err := yamlNodeToAny(&root) + require.NoError(t, err) + doc := docAny.(map[string]any) chains := doc["chains"].(map[string]any) chain := chains["16015286601757825753"].(map[string]any) @@ -241,6 +249,165 @@ chains: require.Equal(t, map[string]any{"quorum": json.Number("1")}, chain["proposer"]) } +func TestYamlNodeToAny_MergeKeyOverrideAfter(t *testing.T) { + t.Parallel() + + yamlContent := ` +base: &base_cfg + qualifier: Base + timelockMinDelay: 0 +chains: + "123": + <<: *base_cfg + qualifier: Override +` + var root yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) + + docAny, err := yamlNodeToAny(&root) + require.NoError(t, err) + chain := docAny.(map[string]any)["chains"].(map[string]any)["123"].(map[string]any) + + require.Equal(t, "Override", chain["qualifier"]) + require.Equal(t, json.Number("0"), chain["timelockMinDelay"]) +} + +func TestYamlNodeToAny_MergeKeyExplicitBeforeMerge(t *testing.T) { + t.Parallel() + + yamlContent := ` +base: &base_cfg + qualifier: Base + timelockMinDelay: 0 +chains: + "123": + qualifier: Override + <<: *base_cfg +` + var root yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) + + docAny, err := yamlNodeToAny(&root) + require.NoError(t, err) + chain := docAny.(map[string]any)["chains"].(map[string]any)["123"].(map[string]any) + + require.Equal(t, "Override", chain["qualifier"]) + require.Equal(t, json.Number("0"), chain["timelockMinDelay"]) +} + +func TestYamlNodeToAny_MergeKeySequence(t *testing.T) { + t.Parallel() + + yamlContent := ` +a: &a + x: 1 +b: &b + y: 2 +m: + <<: [*a, *b] + z: 3 +` + var root yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) + + docAny, err := yamlNodeToAny(&root) + require.NoError(t, err) + m := docAny.(map[string]any)["m"].(map[string]any) + + require.Equal(t, json.Number("1"), m["x"]) + require.Equal(t, json.Number("2"), m["y"]) + require.Equal(t, json.Number("3"), m["z"]) +} + +func TestYamlNodeToAny_MergeKeySequenceConflict(t *testing.T) { + t.Parallel() + + // YAML 1.1: earlier mappings in the merge sequence override later ones. + // https://yaml.org/type/merge.html + yamlContent := ` +a: &a + x: from_a + only_a: 1 +b: &b + x: from_b + only_b: 2 +m: + <<: [*a, *b] +` + var root yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) + + docAny, err := yamlNodeToAny(&root) + require.NoError(t, err) + m := docAny.(map[string]any)["m"].(map[string]any) + + require.Equal(t, "from_a", m["x"]) + require.Equal(t, json.Number("1"), m["only_a"]) + require.Equal(t, json.Number("2"), m["only_b"]) +} + +func TestYamlNodeToAny_QuotedMergeLikeKey(t *testing.T) { + t.Parallel() + + yamlContent := `"<<": 1` + var root yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) + + docAny, err := yamlNodeToAny(&root) + require.NoError(t, err) + doc := docAny.(map[string]any) + + require.Equal(t, json.Number("1"), doc["<<"]) +} + +func TestYamlNodeToAny_InvalidMergeValueSwallowsError(t *testing.T) { + t.Parallel() + + yamlContent := ` +m: + <<: not_a_map + z: 3 +` + var root yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) + + require.Nil(t, YamlNodeToAny(&root)) +} + +func TestYamlNodeToAny_InvalidMergeValue(t *testing.T) { + t.Parallel() + + yamlContent := ` +m: + <<: not_a_map + z: 3 +` + var root yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &root)) + + _, err := yamlNodeToAny(&root) + require.Error(t, err) + require.ErrorContains(t, err, "YAML merge key (<<) value must be a mapping or sequence of mappings") +} + +func TestParseYAMLBytes_InvalidMergeValue(t *testing.T) { + t.Parallel() + + yamlContent := `environment: testnet +domain: ccv +changesets: + - cs1: + payload: + m: + <<: not_a_map + z: 3 +` + _, err := ParseYAMLBytes([]byte(yamlContent)) + require.Error(t, err) + require.ErrorContains(t, err, "failed to decode YAML") + require.ErrorContains(t, err, "YAML merge key (<<) value must be a mapping or sequence of mappings") +} + func TestBuildChangesetInputJSON_AliasChainSelectorKeys(t *testing.T) { t.Parallel()