pkg/settings: add yaml support

jmank88 · jmank88 · commit b7d4fbcbda4e · 2026-02-02T11:42:07.000-06:00
diff --git a/go.mod b/go.mod
@@ -15,6 +15,7 @@ require (
 	github.com/go-json-experiment/json v0.0.0-20250223041408-d3c622f1b874
 	github.com/go-playground/validator/v10 v10.26.0
 	github.com/go-viper/mapstructure/v2 v2.4.0
+	github.com/goccy/go-yaml v1.19.2
 	github.com/golang-jwt/jwt/v5 v5.2.3
 	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
@@ -98,7 +99,6 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
-	github.com/goccy/go-yaml v1.12.0 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/flatbuffers v25.2.10+incompatible // indirect
diff --git a/go.sum b/go.sum
@@ -92,8 +92,8 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
 github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
-github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM=
-github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
+github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
+github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
 github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
diff --git a/pkg/settings/cresettings/defaults.yaml b/pkg/settings/cresettings/defaults.yaml
@@ -0,0 +1,73 @@
+WorkflowLimit: "200"
+WorkflowExecutionConcurrencyLimit: "200"
+GatewayIncomingPayloadSizeLimit: 1mb
+GatewayVaultManagementEnabled: "true"
+VaultCiphertextSizeLimit: 2kb
+VaultIdentifierKeySizeLimit: 64b
+VaultIdentifierOwnerSizeLimit: 64b
+VaultIdentifierNamespaceSizeLimit: 64b
+VaultPluginBatchSizeLimit: "20"
+VaultRequestBatchSizeLimit: "10"
+PerOrg:
+  ZeroBalancePruningTimeout: 24h0m0s
+PerOwner:
+  WorkflowExecutionConcurrencyLimit: "5"
+  VaultSecretsLimit: "100"
+PerWorkflow:
+  TriggerRegistrationsTimeout: 10s
+  TriggerSubscriptionTimeout: 15s
+  TriggerSubscriptionLimit: "10"
+  TriggerEventQueueLimit: "50"
+  TriggerEventQueueTimeout: 10m0s
+  CapabilityConcurrencyLimit: "30"
+  CapabilityCallTimeout: 3m0s
+  SecretsConcurrencyLimit: "5"
+  ExecutionConcurrencyLimit: "5"
+  ExecutionTimeout: 5m0s
+  ExecutionResponseLimit: 100kb
+  WASMMemoryLimit: 100mb
+  WASMBinarySizeLimit: 100mb
+  WASMCompressedBinarySizeLimit: 20mb
+  WASMConfigSizeLimit: 1mb
+  WASMSecretsSizeLimit: 1mb
+  LogLineLimit: 1kb
+  LogEventLimit: "1000"
+  ChainAllowed:
+    Default: "false"
+    Values:
+      "12922642891491394802": "true"
+      "3379446385462418246": "true"
+  CRONTrigger:
+    FastestScheduleInterval: 30s
+  HTTPTrigger:
+    RateLimit: every30s:3
+  LogTrigger:
+    EventRateLimit: every6s:10
+    EventSizeLimit: 5kb
+    FilterAddressLimit: "5"
+    FilterTopicsPerSlotLimit: "10"
+  ChainWrite:
+    TargetsLimit: "10"
+    ReportSizeLimit: 5kb
+    EVM:
+      TransactionGasLimit: "5000000"
+      GasLimit:
+        Default: "5000000"
+        Values:
+          "12922642891491394802": "50000000"
+          "3379446385462418246": "10000000"
+  ChainRead:
+    CallLimit: "15"
+    LogQueryBlockLimit: "100"
+    PayloadSizeLimit: 5kb
+  Consensus:
+    ObservationSizeLimit: 100kb
+    CallLimit: "20"
+  HTTPAction:
+    CallLimit: "5"
+    CacheAgeLimit: 10m0s
+    ConnectionTimeout: 10s
+    RequestSizeLimit: 10kb
+    ResponseSizeLimit: 100kb
+  Secrets:
+    CallLimit: "5"
diff --git a/pkg/settings/cresettings/settings_test.go b/pkg/settings/cresettings/settings_test.go
@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/goccy/go-yaml"
 	"github.com/pelletier/go-toml/v2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -35,6 +36,8 @@ var (
 	defaultsJSON string
 	//go:embed defaults.toml
 	defaultsTOML string
+	//go:embed defaults.yaml
+	defaultsYAML string
 )
 
 func TestDefault(t *testing.T) {
@@ -61,6 +64,22 @@ func TestDefault(t *testing.T) {
 			require.Equal(t, defaultsTOML, string(b))
 		}
 	})
+
+	t.Run("yaml", func(t *testing.T) {
+		jb, err := json.MarshalIndent(Default, "", "\t")
+		if err != nil {
+			log.Fatal(err)
+		}
+		b, err := yaml.JSONToYAML(jb)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if *update {
+			require.NoError(t, os.WriteFile("defaults.yaml", b, 0644))
+		} else {
+			require.Equal(t, defaultsYAML, string(b))
+		}
+	})
 }
 
 func TestSchema_Unmarshal(t *testing.T) {
diff --git a/pkg/settings/json.go b/pkg/settings/json.go
@@ -100,7 +100,7 @@ func readJSONMaps(files fs.FS, dir string) (jsonMap, error) {
 		}
 		m, err := readJSONMap(files, path)
 		if err != nil {
-			return fmt.Errorf("failed to read toml file %s: %w", path, err)
+			return fmt.Errorf("failed to read json file %s: %w", path, err)
 		}
 		name := strings.TrimSuffix(d.Name(), ".json")
 		ms[name] = m
diff --git a/pkg/settings/testdata/config.yaml b/pkg/settings/testdata/config.yaml
@@ -0,0 +1,32 @@
+# File generated from source files. DO NOT EDIT.
+global:
+  Bar:
+    Baz: "10"
+  Foo: "5"
+org:
+  "123":
+    Bar:
+      Baz: "99"
+    Foo: "42"
+  "456":
+    Bar:
+      Baz: "500"
+    Foo: "99"
+owner:
+  00026b4aa7e57ca7b68ae1bf45653f56b656fd3a:
+    Bar:
+      Baz: "200"
+    Foo: "75"
+  8bd112d3f8f92e41c861939545ad387307af9703:
+    Bar:
+      Baz: "43"
+    Foo: "13"
+workflow:
+  15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0:
+    Bar:
+      Baz: "13"
+    Foo: "17"
+  15c631d295ef5e32deb99a10ee6804bc4af1385568f9b3363f6552ac6dbb2cef:
+    Bar:
+      Baz: "50"
+    Foo: "20"
diff --git a/pkg/settings/testdata/yaml/global.yaml b/pkg/settings/testdata/yaml/global.yaml
@@ -0,0 +1,4 @@
+Foo: "5"
+
+Bar:
+  Baz: "10"
diff --git a/pkg/settings/testdata/yaml/org/123.yaml b/pkg/settings/testdata/yaml/org/123.yaml
@@ -0,0 +1,4 @@
+Foo: "42"
+
+Bar:
+  Baz: "99"
diff --git a/pkg/settings/testdata/yaml/org/456.yaml b/pkg/settings/testdata/yaml/org/456.yaml
@@ -0,0 +1,4 @@
+Foo: "99"
+
+Bar:
+  Baz: "500"
diff --git a/pkg/settings/testdata/yaml/owner/00026b4aa7e57ca7b68ae1bf45653f56b656fd3a.yaml b/pkg/settings/testdata/yaml/owner/00026b4aa7e57ca7b68ae1bf45653f56b656fd3a.yaml
@@ -0,0 +1,4 @@
+Foo: "75"
+
+Bar:
+  Baz: "200"
diff --git a/pkg/settings/testdata/yaml/owner/8bd112d3f8f92e41c861939545ad387307af9703.yaml b/pkg/settings/testdata/yaml/owner/8bd112d3f8f92e41c861939545ad387307af9703.yaml
@@ -0,0 +1,4 @@
+Foo: "13"
+
+Bar:
+  Baz: "43"
diff --git a/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0.yaml b/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0.yaml
@@ -0,0 +1,4 @@
+Foo: "17"
+
+Bar:
+  Baz: "13"
diff --git a/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af1385568f9b3363f6552ac6dbb2cef.yaml b/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af1385568f9b3363f6552ac6dbb2cef.yaml
@@ -0,0 +1,4 @@
+Foo: "20"
+
+Bar:
+  Baz: "50"
diff --git a/pkg/settings/yaml.go b/pkg/settings/yaml.go
@@ -0,0 +1,163 @@
+package settings
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/fs"
+	"strconv"
+	"strings"
+
+	"github.com/goccy/go-yaml"
+)
+
+func CombineYAMLFiles(files fs.FS) ([]byte, error) {
+	m := make(map[string]any)
+	global, err := readYAMLMap(files, "global.yaml")
+	if err != nil {
+		return nil, err
+	}
+	if len(global) > 0 {
+		m["global"] = global
+	}
+	orgs, err := readYAMLMaps(files, "org")
+	if err != nil {
+		return nil, err
+	}
+	if len(orgs) > 0 {
+		m["org"] = orgs
+	}
+	owners, err := readYAMLMaps(files, "owner")
+	if err != nil {
+		return nil, err
+	}
+	if len(owners) > 0 {
+		m["owner"] = owners
+	}
+	workflows, err := readYAMLMaps(files, "workflow")
+	if err != nil {
+		return nil, err
+	}
+	if len(workflows) > 0 {
+		m["workflow"] = workflows
+	}
+
+	var b bytes.Buffer
+	b.WriteString(generatedHeader)
+	e := yaml.NewEncoder(&b)
+	err = e.Encode(m)
+	return b.Bytes(), err
+}
+
+func readYAMLMap(files fs.FS, name string) (map[string]any, error) {
+	f, err := files.Open(name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open %s: %w", name, err)
+	}
+	defer f.Close()
+	d := yaml.NewDecoder(f)
+	var m jsonMap
+	err = d.Decode(&m)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse %s: %w", name, err)
+	}
+
+	return m, nil
+}
+
+func readYAMLMaps(files fs.FS, dir string) (jsonMap, error) {
+	ms := make(map[string]any)
+	if err := fs.WalkDir(files, dir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil // ignore
+		}
+		m, err := readYAMLMap(files, path)
+		if err != nil {
+			return fmt.Errorf("failed to read yaml file %s: %w", path, err)
+		}
+		name := strings.TrimSuffix(d.Name(), ".yaml")
+		ms[name] = m
+		return nil
+	}); err != nil {
+		return nil, fmt.Errorf("failed to walk %s: %w", dir, err)
+	}
+	return ms, nil
+}
+
+type yamlSettings struct {
+	m map[string]any
+}
+
+func newYAMLSettings(b []byte) (*yamlSettings, error) {
+	var m map[string]any
+	err := yaml.Unmarshal(b, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &yamlSettings{m: m}, nil
+}
+
+func (y *yamlSettings) getFirst(keys ...string) (string, error) {
+	for _, k := range keys {
+		v, err := y.get(k)
+		if err != nil {
+			return "", err
+		}
+		if v != "" {
+			return v, nil
+		}
+	}
+	return "", nil // no values
+}
+
+func (y *yamlSettings) get(key string) (string, error) {
+	m := y.m
+	parts := strings.Split(key, ".")
+	for i, part := range parts[:len(parts)-1] {
+		v := m[part]
+		if v == nil {
+			return "", nil
+		}
+		var ok bool
+		m, ok = v.(map[string]any)
+		if !ok {
+			return "", fmt.Errorf("invalid key %s: %s is a field", key, strings.Join(parts[:i+1], "."))
+		}
+	}
+
+	field := parts[len(parts)-1]
+	if val, ok := m[field]; ok {
+		switch t := val.(type) {
+		case string:
+			return t, nil
+		case bool:
+			return strconv.FormatBool(t), nil
+		default:
+			return "", fmt.Errorf("non-string value: %s: %t(%v)", key, val, val)
+		}
+	}
+	return "", nil // no value
+}
+
+type yamlGetter struct {
+	settings *yamlSettings
+}
+
+func NewYAMLGetter(b []byte) (Getter, error) {
+	s, err := newYAMLSettings(b)
+	if err != nil {
+		return nil, err
+	}
+	return &yamlGetter{settings: s}, nil
+}
+
+func (y *yamlGetter) GetScoped(ctx context.Context, scope Scope, key string) (value string, err error) {
+	keys, err := scope.rawKeys(ctx, key)
+	if err != nil {
+		return "", fmt.Errorf("failed to get raw keys: %w", err)
+	}
+	return y.settings.getFirst(keys...)
+}
diff --git a/pkg/settings/yaml_test.go b/pkg/settings/yaml_test.go

Original file line number	Diff line number	Diff line change
`@@ -100,7 +100,7 @@ func readJSONMaps(files fs.FS, dir string) (jsonMap, error) {`
`100`	`100`	`}`
`101`	`101`	`m, err := readJSONMap(files, path)`
`102`	`102`	`if err != nil {`
`103`		`- return fmt.Errorf("failed to read toml file %s: %w", path, err)`
	`103`	`+ return fmt.Errorf("failed to read json file %s: %w", path, err)`
`104`	`104`	`}`
`105`	`105`	`name := strings.TrimSuffix(d.Name(), ".json")`
`106`	`106`	`ms[name] = m`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +Foo: "5"
++
 +Bar:
 +  Baz: "10"