Add common code to encode and decode metadata.

Signed-off-by: Felix Fontein <felix@fontein.de>

Author: felixfontein

stores/dotenv/store.go

@@ -3,7 +3,6 @@ package dotenv //import "github.com/getsops/sops/v3/stores/dotenv"
 import (
 	"bytes"
 	"fmt"
-	"sort"
 	"strings"
 
 	"github.com/getsops/sops/v3"
@@ -33,43 +32,15 @@ func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) {
 	if err != nil {
 		return sops.Tree{}, err
 	}
-
-	var resultBranch sops.TreeBranch
-	mdMap := make(map[string]interface{})
-	for _, item := range branches[0] {
-		switch key := item.Key.(type) {
-		case string:
-			if strings.HasPrefix(key, SopsPrefix) {
-				key = key[len(SopsPrefix):]
-				mdMap[key] = item.Value
-			} else {
-				resultBranch = append(resultBranch, item)
-			}
-		case sops.Comment:
-			resultBranch = append(resultBranch, item)
-		default:
-			panic(fmt.Sprintf("Unexpected type: %T (value %#v)", key, key))
-		}
-	}
-	if len(mdMap) == 0 {
-		return sops.Tree{}, sops.MetadataNotFound
-	}
-
-	stores.DecodeNewLines(mdMap)
-	metadata, err := stores.UnflattenMetadata(mdMap)
+	branches, metadata, err := stores.ExtractMetadata(branches, stores.MetadataOpts{
+		Flatten: stores.MetadataFlattenFull,
+	})
 	if err != nil {
 		return sops.Tree{}, err
 	}
-	internalMetadata, err := metadata.ToInternal()
-	if err != nil {
-		return sops.Tree{}, err
-	}
-
 	return sops.Tree{
-		Branches: sops.TreeBranches{
-			resultBranch,
-		},
-		Metadata: internalMetadata,
+		Branches: branches,
+		Metadata: metadata,
 	}, nil
 }
 
@@ -107,29 +78,13 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) {
 // EmitEncryptedFile returns the encrypted file's bytes corresponding to a sops
 // runtime object
 func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) {
-	metadata := stores.MetadataFromInternal(in.Metadata)
-	mdItems, err := stores.FlattenMetadata(metadata)
+	branches, err := stores.SerializeMetadata(in, stores.MetadataOpts{
+		Flatten: stores.MetadataFlattenFull,
+	})
 	if err != nil {
-		return nil, err
-	}
-
-	stores.EncodeNonStrings(mdItems)
-	stores.EncodeNewLines(mdItems)
-
-	var keys []string
-	for k := range mdItems {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	for _, key := range keys {
-		var value = mdItems[key]
-		if value == nil {
-			continue
-		}
-		in.Branches[0] = append(in.Branches[0], sops.TreeItem{Key: SopsPrefix + key, Value: value})
+		return nil, fmt.Errorf("Error marshaling metadata: %s", err)
 	}
-	return store.EmitPlainFile(in.Branches)
+	return store.EmitPlainFile(branches)
 }
 
 // EmitPlainFile returns the plaintext file's bytes corresponding to a sops

stores/flatten.go

@@ -2,10 +2,14 @@ package stores
 
 import (
 	"fmt"
+	"reflect"
+	"slices"
 	"strconv"
 	"strings"
 
 	"github.com/go-viper/mapstructure/v2"
+
+	"github.com/getsops/sops/v3"
 )
 
 const mapSeparator = "__map_"
@@ -270,3 +274,154 @@ func EncodeNonStrings(m map[string]interface{}) {
 		}
 	}
 }
+
+func descendMap(branch sops.TreeBranch, key string) (sops.TreeBranch, reflect.Value) {
+	for idx, elt := range branch {
+		if elt.Key == key {
+			return branch, reflect.ValueOf(&branch[idx].Value)
+		}
+	}
+	newBranch := append(branch, sops.TreeItem{
+		Key:   key,
+		Value: nil,
+	})
+	return newBranch, reflect.ValueOf(&newBranch[len(newBranch)-1].Value)
+}
+
+func descendList(array []interface{}, index int) ([]interface{}, reflect.Value) {
+	if index >= len(array) {
+		if index >= cap(array) {
+			array = slices.Grow(array, index+1-cap(array))
+		}
+		array = array[:index+1]
+	}
+	return array, reflect.ValueOf(&array[index])
+}
+
+func descend(value reflect.Value, t token) (reflect.Value, error) {
+	interfaceType := reflect.TypeOf((*interface{})(nil))
+	switch currToken := t.(type) {
+	case mapToken:
+		// This special case is only needed for the root
+		treeBranchPtrType := reflect.TypeOf((*sops.TreeBranch)(nil))
+		if value.Type() == treeBranchPtrType {
+			v := *value.Interface().(*sops.TreeBranch)
+			v, nextVal := descendMap(v, currToken.key)
+			reflect.Indirect(value).Set(reflect.ValueOf(v))
+			return nextVal, nil
+		}
+		if value.Type() == interfaceType {
+			val := reflect.Indirect(value)
+			if val.IsNil() {
+				v, nextVal := descendMap(nil, currToken.key)
+				val.Set(reflect.ValueOf(v))
+				return nextVal, nil
+			}
+			if v, ok := val.Interface().(sops.TreeBranch); ok {
+				v, nextVal := descendMap(v, currToken.key)
+				val.Set(reflect.ValueOf(v))
+				return nextVal, nil
+			}
+		}
+		return reflect.Value{}, fmt.Errorf("Type mismatch: can only use string key for map")
+	case listToken:
+		if value.Type() == interfaceType {
+			val := reflect.Indirect(value)
+			if val.IsNil() {
+				v, nextVal := descendList(nil, currToken.position)
+				val.Set(reflect.ValueOf(v))
+				return nextVal, nil
+			}
+			if v, ok := val.Interface().([]interface{}); ok {
+				v, nextVal := descendList(v, currToken.position)
+				val.Set(reflect.ValueOf(v))
+				return nextVal, nil
+			}
+		}
+		return reflect.Value{}, fmt.Errorf("Type mismatch: can only use integer key for list")
+	default:
+		return reflect.Value{}, fmt.Errorf("Internal error: unknown token %q", t)
+	}
+}
+
+func unflattenTreeBranch(branch sops.TreeBranch) (sops.TreeBranch, error) {
+	var result sops.TreeBranch
+	for _, item := range branch {
+		if _, ok := item.Key.(sops.Comment); ok {
+			continue
+		}
+		if key, ok := item.Key.(string); ok {
+			current := reflect.ValueOf(&result)
+			tokens := tokenize(key)
+			for _, token := range tokens {
+				var err error
+				current, err = descend(current, token)
+				if err != nil {
+					return nil, fmt.Errorf("Error while unflattening %q: %w", key, err)
+				}
+			}
+			reflect.Indirect(current).Set(reflect.ValueOf(item.Value))
+			continue
+		} else {
+			return nil, fmt.Errorf("Found non-string key %q when unflattening", item.Key)
+		}
+	}
+	return result, nil
+}
+
+func flattenDescendValue(value interface{}, key string, destination sops.TreeBranch, destinationMap *map[string]bool) (sops.TreeBranch, error) {
+	switch value := value.(type) {
+	case sops.TreeBranch:
+		return flattenDescendMap(value, key+mapSeparator, destination, destinationMap)
+	case []interface{}:
+		return flattenDescendArray(value, key+listSeparator, destination, destinationMap)
+	}
+	if _, ok := (*destinationMap)[key]; ok {
+		return nil, fmt.Errorf("Found key collision %q while flattening", key)
+	}
+	destination = append(destination, sops.TreeItem{
+		Key:   key,
+		Value: value,
+	})
+	(*destinationMap)[key] = true
+	return destination, nil
+}
+
+func flattenDescendMap(branch sops.TreeBranch, prefix string, destination sops.TreeBranch, destinationMap *map[string]bool) (sops.TreeBranch, error) {
+	for _, item := range branch {
+		if _, ok := item.Key.(sops.Comment); ok {
+			continue
+		}
+		if key, ok := item.Key.(string); ok {
+			var err error
+			destination, err = flattenDescendValue(item.Value, prefix+key, destination, destinationMap)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, fmt.Errorf("Found non-string key %q when flattening", item.Key)
+		}
+	}
+	return destination, nil
+}
+
+func flattenDescendArray(array []interface{}, prefix string, destination sops.TreeBranch, destinationMap *map[string]bool) (sops.TreeBranch, error) {
+	i := 0
+	for _, item := range array {
+		if _, ok := item.(sops.Comment); ok {
+			continue
+		}
+		var err error
+		destination, err = flattenDescendValue(item, fmt.Sprintf("%s%d", prefix, i), destination, destinationMap)
+		if err != nil {
+			return nil, err
+		}
+		i += 1
+	}
+	return destination, nil
+}
+
+func flattenTreeBranch(branch sops.TreeBranch, prefix string) (sops.TreeBranch, error) {
+	destinationMap := map[string]bool{}
+	return flattenDescendMap(branch, prefix, nil, &destinationMap)
+}