diff --git a/CHANGELOG.md b/CHANGELOG.md index 332ac639..a7dd4b2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Fix YAML output escaping emoji and supplementary-plane Unicode characters (U+10000–U+10FFFF) to `\UNNNNNNNN` sequences instead of preserving UTF-8 ([#437](https://github.com/TomWright/dasel/issues/437)). + ## [v3.11.0] - 2026-05-19 ### Added diff --git a/parsing/yaml/yaml_test.go b/parsing/yaml/yaml_test.go index f87ac49f..d80d5306 100644 --- a/parsing/yaml/yaml_test.go +++ b/parsing/yaml/yaml_test.go @@ -674,6 +674,22 @@ single: 'Tom' `, }.run) + t.Run("emoji preserved as UTF-8", rwTestCase{ + in: "US: \"\U0001F1FA\U0001F1F8\"\n", + }.run) + + t.Run("simple emoji preserved", rwTestCase{ + in: "face: \"\U0001F600\"\n", + }.run) + + t.Run("mixed ASCII and emoji", rwTestCase{ + in: "msg: \"hello \U0001F30D world\"\n", + }.run) + + t.Run("supplementary plane unicode", rwTestCase{ + in: "clef: \"\U0001D11E\"\ncjk: \"\U00020000\"\nmath: \"\U0001D54F\"\n", + }.run) + t.Run("yaml expansion budget resets per document", func(t *testing.T) { reader, err := parsing.Format("yaml").NewReader(parsing.DefaultReaderOptions()) if err != nil { diff --git a/parsing/yaml/yaml_writer.go b/parsing/yaml/yaml_writer.go index c56e6138..1b838fa0 100644 --- a/parsing/yaml/yaml_writer.go +++ b/parsing/yaml/yaml_writer.go @@ -1,7 +1,10 @@ package yaml import ( + "bytes" + "encoding/hex" "fmt" + "unicode/utf8" "github.com/tomwright/dasel/v3/model" "github.com/tomwright/dasel/v3/parsing" @@ -29,7 +32,52 @@ func (j *yamlWriter) Write(value *model.Value) ([]byte, error) { if err != nil { return nil, err } - return yaml.Marshal(res) + out, err := yaml.Marshal(res) + if err != nil { + return nil, err + } + return unescapeYAMLUnicode(out), nil +} + +// unescapeYAMLUnicode replaces \UNNNNNNNN escape sequences (8 hex digits) with +// the corresponding UTF-8 bytes. The yaml library incorrectly escapes +// supplementary-plane characters (U+10000–U+10FFFF) because its isPrintable +// function does not handle 4-byte UTF-8 sequences. +func unescapeYAMLUnicode(data []byte) []byte { + marker := []byte(`\U`) + var result []byte + for { + idx := bytes.Index(data, marker) + if idx == -1 { + break + } + // Need exactly 8 hex digits after \U + if idx+10 > len(data) { + result = append(result, data[:idx+2]...) + data = data[idx+2:] + continue + } + hexBytes := data[idx+2 : idx+10] + decoded, err := hex.DecodeString(string(hexBytes)) + if err != nil || len(decoded) != 4 { + result = append(result, data[:idx+2]...) + data = data[idx+2:] + continue + } + r := rune(decoded[0])<<24 | rune(decoded[1])<<16 | rune(decoded[2])<<8 | rune(decoded[3]) + if r < 0x10000 || r > 0x10FFFF || !utf8.ValidRune(r) { + result = append(result, data[:idx+10]...) + data = data[idx+10:] + continue + } + result = append(result, data[:idx]...) + var buf [4]byte + n := utf8.EncodeRune(buf[:], r) + result = append(result, buf[:n]...) + data = data[idx+10:] + } + result = append(result, data...) + return result } func (yv *yamlValue) ToNode() (*yaml.Node, error) {