Merge branch 'feature/restore-configs-projection' into 'master'

feat: expose logicalRestore configs in config projection and UI

Closes #701

See merge request postgres-ai/database-lab!1130

Author: agneum

engine/pkg/models/configuration.go

@@ -24,6 +24,7 @@ type ConfigProjection struct {
 	DBList               map[string]interface{} `proj:"retrieval.spec.logicalDump.options.databases,createKey"`
 	DumpParallelJobs     *int64                 `proj:"retrieval.spec.logicalDump.options.parallelJobs"`
 	RestoreParallelJobs  *int64                 `proj:"retrieval.spec.logicalRestore.options.parallelJobs"`
+	RestoreConfigs       map[string]interface{} `proj:"retrieval.spec.logicalRestore.options.configs,createKey"`
 	DumpCustomOptions    []interface{}          `proj:"retrieval.spec.logicalDump.options.customOptions"`
 	RestoreCustomOptions []interface{}          `proj:"retrieval.spec.logicalRestore.options.customOptions"`
 	IgnoreDumpErrors     *bool                  `proj:"retrieval.spec.logicalDump.options.ignoreErrors"`

engine/pkg/util/projection/load_yaml_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
 )
 
 func TestLoadYaml(t *testing.T) {
@@ -28,3 +29,33 @@ func TestLoadYamlNull(t *testing.T) {
 
 	requireEmpty(t, s)
 }
+
+func TestLoadYaml_MergeKey(t *testing.T) {
+	type mergeStruct struct {
+		Configs      map[string]interface{} `proj:"parent.options.configs"`
+		ParallelJobs *int64                 `proj:"parent.options.parallelJobs"`
+	}
+
+	const yamlData = `
+defaults: &defaults
+  configs:
+    shared_buffers: 1GB
+    work_mem: 100MB
+
+parent:
+  options:
+    <<: *defaults
+    parallelJobs: 4
+`
+
+	node := &yaml.Node{}
+	err := yaml.Unmarshal([]byte(yamlData), node)
+	require.NoError(t, err)
+
+	s := &mergeStruct{}
+	err = LoadYaml(s, node, LoadOptions{})
+	require.NoError(t, err)
+
+	require.Equal(t, map[string]interface{}{"shared_buffers": "1GB", "work_mem": "100MB"}, s.Configs)
+	require.Equal(t, int64(4), *s.ParallelJobs)
+}

engine/pkg/util/projection/store_yaml_test.go

@@ -184,6 +184,51 @@ nested:
 	require.Equal(t, []interface{}{"--no-privileges", "--no-owner"}, itemsVal)
 }
 
+func TestStoreYaml_MergeKeyCreatesDirect(t *testing.T) {
+	type mergeStruct struct {
+		Configs      map[string]interface{} `proj:"parent.options.configs,createKey"`
+		ParallelJobs *int64                 `proj:"parent.options.parallelJobs"`
+	}
+
+	const yamlData = `
+defaults: &defaults
+  configs:
+    shared_buffers: 1GB
+    work_mem: 100MB
+
+parent:
+  options:
+    <<: *defaults
+    parallelJobs: 4
+`
+
+	node := &yaml.Node{}
+	err := yaml.Unmarshal([]byte(yamlData), node)
+	require.NoError(t, err)
+
+	pj := int64(8)
+	s := &mergeStruct{Configs: map[string]interface{}{"fsync": "off", "maintenance_work_mem": "8GB"}, ParallelJobs: &pj}
+
+	err = StoreYaml(s, node, StoreOptions{})
+	require.NoError(t, err)
+
+	soft, err := NewSoftYaml(node)
+	require.NoError(t, err)
+
+	configVal, err := soft.Get(FieldGet{Path: []string{"parent", "options", "configs"}, Type: ptypes.Map})
+	require.NoError(t, err)
+	require.Equal(t, map[string]interface{}{"fsync": "off", "maintenance_work_mem": "8GB"}, configVal)
+
+	pjVal, err := soft.Get(FieldGet{Path: []string{"parent", "options", "parallelJobs"}, Type: ptypes.Int64})
+	require.NoError(t, err)
+	require.Equal(t, int64(8), pjVal)
+
+	// verify anchor source was not modified
+	defaultConfigs, err := soft.Get(FieldGet{Path: []string{"defaults", "configs"}, Type: ptypes.Map})
+	require.NoError(t, err)
+	require.Equal(t, map[string]interface{}{"shared_buffers": "1GB", "work_mem": "100MB"}, defaultConfigs)
+}
+
 func TestStoreYaml_NilMapPreservesExisting(t *testing.T) {
 	type mapStruct struct {
 		Name   *string                `proj:"nested.name"`

engine/pkg/util/projection/yaml.go

@@ -42,9 +42,21 @@ func (y *yamlSoft) Set(set FieldSet) error {
 			return fmt.Errorf("node is not a mapping node")
 		}
 
-		child, hasChild := findNodeForKey(node, key)
+		isLastSegment := i == len(set.Path)-1
+
+		var (
+			child    *yaml.Node
+			hasChild bool
+		)
+
+		if isLastSegment && set.CreateKey {
+			child, hasChild = findDirectNodeForKey(node, key)
+		} else {
+			child, hasChild = findNodeForKey(node, key)
+		}
+
 		if !hasChild {
-			if set.CreateKey && i == len(set.Path)-1 {
+			if set.CreateKey && isLastSegment {
 				child = &yaml.Node{
 					Kind: yaml.ScalarNode,
 					Tag:  "!!map",
@@ -129,7 +141,8 @@ func (y *yamlSoft) Get(get FieldGet) (interface{}, error) {
 	return typed, nil
 }
 
-func findNodeForKey(node *yaml.Node, key string) (*yaml.Node, bool) {
+// findDirectNodeForKey looks up a key in a mapping node's direct children only.
+func findDirectNodeForKey(node *yaml.Node, key string) (*yaml.Node, bool) {
 	for i := 0; i < len(node.Content); i += 2 {
 		if node.Content[i].Value == key {
 			return node.Content[i+1], true
@@ -139,6 +152,34 @@ func findNodeForKey(node *yaml.Node, key string) (*yaml.Node, bool) {
 	return nil, false
 }
 
+// findNodeForKey looks up a key in a mapping node, resolving YAML merge keys (<<: *alias).
+func findNodeForKey(node *yaml.Node, key string) (*yaml.Node, bool) {
+	if child, ok := findDirectNodeForKey(node, key); ok {
+		return child, true
+	}
+
+	for i := 0; i < len(node.Content); i += 2 {
+		if node.Content[i].Tag != "!!merge" {
+			continue
+		}
+
+		merged := node.Content[i+1]
+		if merged.Kind == yaml.AliasNode && merged.Alias != nil {
+			merged = merged.Alias
+		}
+
+		if merged.Kind != yaml.MappingNode {
+			continue
+		}
+
+		if child, ok := findNodeForKey(merged, key); ok {
+			return child, true
+		}
+	}
+
+	return nil, false
+}
+
 func convertMap(node *yaml.Node) (map[string]interface{}, error) {
 	convertedMap := make(map[string]interface{}, 0)
 

ui/packages/ce/src/api/configs/updateConfig.ts

@@ -2,6 +2,7 @@ import {
   postUniqueCustomOptions,
   postUniqueDatabases,
 } from '@postgres.ai/shared/pages/Instance/Configuration/utils'
+import { formatTuningParamsToObj } from '@postgres.ai/shared/types/api/endpoints/testDbSource'
 import { Config } from '@postgres.ai/shared/types/api/entities/config'
 import { request } from 'helpers/request'
 
@@ -51,6 +52,7 @@ export const updateConfig = async (req: Config) => {
               ),
               parallelJobs: req.restoreParallelJobs,
               ignoreErrors: req.restoreIgnoreErrors,
+              configs: formatTuningParamsToObj(req.restoreConfigs),
             },
           },
         },

ui/packages/shared/pages/Instance/Configuration/index.tsx

@@ -1120,6 +1120,16 @@ export const Configuration = observer(
                     )
                   }
                 />
+                <InputWithTooltip
+                  type="textarea"
+                  label="Restore PostgreSQL configs"
+                  value={formik.values.restoreConfigs}
+                  tooltipText={tooltipText.restoreConfigs}
+                  disabled={isConfigurationDisabled}
+                  onChange={(e) =>
+                    formik.setFieldValue('restoreConfigs', e.target.value)
+                  }
+                />
                 <FormControlLabel
                   style={{ maxWidth: 'max-content' }}
                   control={

ui/packages/shared/pages/Instance/Configuration/tooltipText.tsx

@@ -138,6 +138,26 @@ export const tooltipText = {
       hyphen, underscore, equal sign, and double quotes.
     </div>
   ),
+  restoreConfigs: () => (
+    <div>
+      PostgreSQL configuration parameters applied during logical restore (one{' '}
+      <span className={styles.firaCodeFont}>parameter=value</span> per line).
+      These settings are written to{' '}
+      <span className={styles.firaCodeFont}>postgresql.conf</span> before
+      restore starts and do not affect clones. Useful for tuning restore
+      performance, for example:
+      <br />
+      <span className={styles.firaCodeFont}>maintenance_work_mem=8GB</span>
+      <br />
+      <span className={styles.firaCodeFont}>
+        max_parallel_maintenance_workers=7
+      </span>
+      <br />
+      <span className={styles.firaCodeFont}>shared_preload_libraries=</span>
+      <br />
+      <span className={styles.firaCodeFont}>fsync=off</span>
+    </div>
+  ),
   timetable: () => (
     <div>
       Schedule for full data refreshes, in{' '}

ui/packages/shared/pages/Instance/Configuration/useForm.ts

@@ -28,6 +28,7 @@ export type FormValues = {
   dumpIgnoreErrors: boolean
   restoreParallelJobs: string
   restoreIgnoreErrors: boolean
+  restoreConfigs: string
   pgDumpCustomOptions: string
   pgRestoreCustomOptions: string
 }
@@ -60,6 +61,7 @@ export const useForm = (onSubmit: (values: FormValues) => void) => {
       databases: '',
       dumpParallelJobs: '',
       restoreParallelJobs: '',
+      restoreConfigs: '',
       pgDumpCustomOptions: '',
       pgRestoreCustomOptions: '',
       dumpIgnoreErrors: false,

ui/packages/shared/types/api/endpoints/testDbSource.ts

@@ -39,8 +39,10 @@ export const formatTuningParamsToObj = (tuningParams: string | undefined) => {
   if (tuningParams) {
     const tuningParamsArr = tuningParams.split('\n')
     tuningParamsArr.forEach((param) => {
-      const paramArr = param.split('=')
-      formattedTuningParams[paramArr[0]] = paramArr[1]
+      const eqIndex = param.indexOf('=')
+      if (eqIndex !== -1) {
+        formattedTuningParams[param.substring(0, eqIndex)] = param.substring(eqIndex + 1)
+      }
     })
   }
 

ui/packages/shared/types/api/entities/config.ts

@@ -51,6 +51,7 @@ export type configTypes = {
           customOptions?: string[]
           parallelJobs?: string | number
           ignoreErrors?: boolean
+          configs?: { [key: string]: string }
         }
       }
     }
@@ -103,6 +104,14 @@ export const formatConfig = (config: configTypes) => {
       config.retrieval?.spec?.logicalRestore?.options?.parallelJobs,
     restoreIgnoreErrors:
       config.retrieval?.spec?.logicalRestore?.options?.ignoreErrors,
+    restoreConfigs: (() => {
+      const configs =
+        config.retrieval?.spec?.logicalRestore?.options?.configs
+      if (!configs || Object.keys(configs).length === 0) return ''
+      return Object.entries(configs)
+        .map(([k, v]) => `${k}=${v}`)
+        .join('\n')
+    })(),
     pgDumpCustomOptions: formatDumpCustomOptions(
       (config.retrieval?.spec?.logicalDump?.options
         ?.customOptions as string[] | undefined) ?? null,