diff --git a/go.mod b/go.mod index e684897be3..3f7c4c6606 100644 --- a/go.mod +++ b/go.mod @@ -405,3 +405,5 @@ tool ( gotest.tools/gotestsum mvdan.cc/gofumpt ) + +replace github.com/keboola/keboola-sdk-go/v2 => github.com/keboola/keboola-sdk-go/v2 v2.6.7-0.20251112071047-336eaa503e98 diff --git a/go.sum b/go.sum index 22bea79f61..a0d9914ca0 100644 --- a/go.sum +++ b/go.sum @@ -593,8 +593,8 @@ github.com/keboola/go-oauth2-proxy/v7 v7.0.0-20251107090355-631a3f56b65f h1:8twG github.com/keboola/go-oauth2-proxy/v7 v7.0.0-20251107090355-631a3f56b65f/go.mod h1:inTUfQiBS2zSsGRH7XzTkSs4iOoz1RWBb9qT2GL3XwY= github.com/keboola/go-utils v1.4.0 h1:WTyj95yrr8O8HxtC8TSTyUcElZiRGDeEdVvDpFo6HUo= github.com/keboola/go-utils v1.4.0/go.mod h1:IopwJzFz2gh0Yj3fUbIe2eamRoDKzbXvjqFjQyw3ZdQ= -github.com/keboola/keboola-sdk-go/v2 v2.6.6 h1:z7orJH13yp91LsvGRcvGcmlFil6ezD9As8kNJHaN6Ts= -github.com/keboola/keboola-sdk-go/v2 v2.6.6/go.mod h1:f1j2d3Iii+nAHEJ9yDfdg2bKccU6kUYfuiIwKLT2YyQ= +github.com/keboola/keboola-sdk-go/v2 v2.6.7-0.20251112071047-336eaa503e98 h1:ERxC2n321DISY9YMmVtH7sFmHwupUJNGJwcPwQsBHqA= +github.com/keboola/keboola-sdk-go/v2 v2.6.7-0.20251112071047-336eaa503e98/go.mod h1:f1j2d3Iii+nAHEJ9yDfdg2bKccU6kUYfuiIwKLT2YyQ= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/internal/pkg/project/manifest/file.go b/internal/pkg/project/manifest/file.go index e8aaf5d1c0..918b044c81 100644 --- a/internal/pkg/project/manifest/file.go +++ b/internal/pkg/project/manifest/file.go @@ -34,6 +34,7 @@ type file struct { AllowedBranches model.AllowedBranches `json:"allowedBranches" validate:"required,min=1"` IgnoredComponents model.ComponentIDs `json:"ignoredComponents"` Templates Templates `json:"templates"` + Vault Vault `json:"vault,omitempty"` Branches []*model.BranchManifest `json:"branches" validate:"dive"` Configs []*model.ConfigManifestWithRows `json:"configurations" validate:"dive"` } @@ -42,6 +43,10 @@ type Templates struct { Repositories []model.TemplateRepository `json:"repositories,omitempty" validate:"dive"` } +type Vault struct { + Variables []*keboola.VaultVariable `json:"variables,omitempty" validate:"dive"` +} + func newFile(projectID keboola.ProjectID, apiHost string) *file { return &file{ Version: build.MajorVersion, diff --git a/internal/pkg/project/manifest/manifest.go b/internal/pkg/project/manifest/manifest.go index 2bc302007b..edc9c0afed 100644 --- a/internal/pkg/project/manifest/manifest.go +++ b/internal/pkg/project/manifest/manifest.go @@ -46,11 +46,12 @@ type Manifest struct { // allowTargetENV allows usage KBC_PROJECT_ID and KBC_BRANCH_ID envs to override manifest values allowTargetENV bool // mapping between manifest representation and memory representation - mapping []mappingItem - project Project - naming naming.Template - filter model.ObjectsFilter - repositories []model.TemplateRepository + mapping []mappingItem + project Project + naming naming.Template + filter model.ObjectsFilter + repositories []model.TemplateRepository + vaultVariables []*keboola.VaultVariable } type Project struct { @@ -157,6 +158,7 @@ func Load(ctx context.Context, logger log.Logger, fs filesystem.Fs, envs env.Pro m.filter.SetAllowedBranches(content.AllowedBranches) m.filter.SetIgnoredComponents(content.IgnoredComponents) m.repositories = content.Templates.Repositories + m.vaultVariables = content.Vault.Variables // Set records if err := m.SetRecords(content.records()); err != nil && !ignoreErrors { @@ -176,6 +178,7 @@ func (m *Manifest) Save(ctx context.Context, fs filesystem.Fs) error { content.AllowedBranches = m.filter.AllowedBranches() content.IgnoredComponents = m.filter.IgnoredComponents() content.Templates.Repositories = m.repositories + content.Vault.Variables = m.vaultVariables content.setRecords(m.All()) // Map memory IDs to manifest IDs @@ -267,3 +270,25 @@ func (m *Manifest) TemplateRepository(name string) (model.TemplateRepository, bo } return model.TemplateRepository{}, false } + +func (m *Manifest) VaultVariables() []*keboola.VaultVariable { + return m.vaultVariables +} + +func (m *Manifest) SetVaultVariables(variables []*keboola.VaultVariable) { + m.vaultVariables = variables +} + +func (m *Manifest) AddVaultVariable(variable *keboola.VaultVariable) { + m.vaultVariables = append(m.vaultVariables, variable) +} + +func (m *Manifest) RemoveVaultVariable(hash keboola.VaultVariableHash) bool { + for i, v := range m.vaultVariables { + if v.Hash == hash { + m.vaultVariables = append(m.vaultVariables[:i], m.vaultVariables[i+1:]...) + return true + } + } + return false +} diff --git a/internal/pkg/service/cli/cmd/remote/cmd.go b/internal/pkg/service/cli/cmd/remote/cmd.go index 8dfe1fc2d9..2ccaf9b50a 100644 --- a/internal/pkg/service/cli/cmd/remote/cmd.go +++ b/internal/pkg/service/cli/cmd/remote/cmd.go @@ -7,6 +7,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/cli/cmd/remote/file" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/cmd/remote/job" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/cmd/remote/table" + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/cmd/remote/vault" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/cmd/remote/workspace" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/helpmsg" @@ -24,6 +25,7 @@ func Commands(p dependencies.Provider) *cobra.Command { job.Commands(p), workspace.Commands(p), table.Commands(p), + vault.Commands(p), ) return cmd diff --git a/internal/pkg/service/cli/cmd/remote/vault/cmd.go b/internal/pkg/service/cli/cmd/remote/vault/cmd.go new file mode 100644 index 0000000000..6c6d369772 --- /dev/null +++ b/internal/pkg/service/cli/cmd/remote/vault/cmd.go @@ -0,0 +1,24 @@ +package vault + +import ( + "github.com/spf13/cobra" + + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/cmd/remote/vault/create" + deleteVault "github.com/keboola/keboola-as-code/internal/pkg/service/cli/cmd/remote/vault/delete" + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies" + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/helpmsg" +) + +func Commands(p dependencies.Provider) *cobra.Command { + cmd := &cobra.Command{ + Use: `vault`, + Short: helpmsg.Read(`remote/vault/short`), + Long: helpmsg.Read(`remote/vault/long`), + Hidden: true, + } + cmd.AddCommand( + create.Command(p), + deleteVault.Command(p), + ) + return cmd +} diff --git a/internal/pkg/service/cli/cmd/remote/vault/create/cmd.go b/internal/pkg/service/cli/cmd/remote/vault/create/cmd.go new file mode 100644 index 0000000000..73147d88a5 --- /dev/null +++ b/internal/pkg/service/cli/cmd/remote/vault/create/cmd.go @@ -0,0 +1,91 @@ +package create + +import ( + "github.com/spf13/cobra" + + "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" + + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies" + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/helpmsg" + "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" + "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + saveManifest "github.com/keboola/keboola-as-code/pkg/lib/operation/project/local/manifest/save" +) + +type Flags struct { + StorageAPIHost configmap.Value[string] `configKey:"storage-api-host" configShorthand:"H" configUsage:"storage API host, eg. \"connection.keboola.com\""` + StorageAPIToken configmap.Value[string] `configKey:"storage-api-token" configShorthand:"t" configUsage:"storage API token from your project"` + Name configmap.Value[string] `configKey:"name" configUsage:"name of the vault variable"` +} + +func DefaultFlags() Flags { + return Flags{} +} + +func Command(p dependencies.Provider) *cobra.Command { + cmd := &cobra.Command{ + Use: `create [name]`, + Short: helpmsg.Read(`remote/vault/create/short`), + Long: helpmsg.Read(`remote/vault/create/long`), + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) (cmdErr error) { + f := Flags{} + if err := p.BaseScope().ConfigBinder().Bind(cmd.Context(), cmd.Flags(), args, &f); err != nil { + return err + } + + d, err := p.RemoteCommandScope(cmd.Context(), f.StorageAPIHost, f.StorageAPIToken) + if err != nil { + return err + } + + logger := d.Logger() + + prj, _, err := d.LocalProject(cmd.Context(), false) + if err != nil { + return err + } + + var variableName string + if len(args) > 0 { + variableName = args[0] + } else if f.Name.Value != "" { + variableName = f.Name.Value + } else { + variableName, _ = d.Dialogs().Ask(AskVariableName()) + } + + variableValue, _ := d.Dialogs().Ask(AskVariableValue()) + + defer d.EventSender().SendCmdEvent(cmd.Context(), d.Clock().Now(), &cmdErr, "remote-vault-create") + + logger.Infof(cmd.Context(), "Creating vault variable \"%s\"...", variableName) + + payload := &keboola.VaultVariableCreatePayload{ + Key: variableName, + Value: variableValue, + } + + variable, err := d.KeboolaProjectAPI().CreateVariableRequest(payload).Send(cmd.Context()) + if err != nil { + return errors.Errorf("failed to create vault variable: %w", err) + } + + logger.Infof(cmd.Context(), "Vault variable \"%s\" created with hash: %s", variableName, variable.Hash) + + prj.ProjectManifest().AddVaultVariable(variable) + + if _, err := saveManifest.Run(cmd.Context(), prj.ProjectManifest(), prj.Fs(), d); err != nil { + return errors.Errorf("failed to save manifest: %w", err) + } + + logger.Info(cmd.Context(), "Manifest updated successfully.") + + return nil + }, + } + + configmap.MustGenerateFlags(cmd.Flags(), DefaultFlags()) + + return cmd +} diff --git a/internal/pkg/service/cli/cmd/remote/vault/create/dialog.go b/internal/pkg/service/cli/cmd/remote/vault/create/dialog.go new file mode 100644 index 0000000000..24b7d2a755 --- /dev/null +++ b/internal/pkg/service/cli/cmd/remote/vault/create/dialog.go @@ -0,0 +1,22 @@ +package create + +import ( + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/prompt" +) + +func AskVariableName() *prompt.Question { + return &prompt.Question{ + Label: "Variable name", + Description: "Enter the vault variable name.", + Validator: prompt.ValueRequired, + } +} + +func AskVariableValue() *prompt.Question { + return &prompt.Question{ + Label: "Variable value", + Description: "Enter the vault variable value.", + Validator: prompt.ValueRequired, + Hidden: true, + } +} diff --git a/internal/pkg/service/cli/cmd/remote/vault/delete/cmd.go b/internal/pkg/service/cli/cmd/remote/vault/delete/cmd.go new file mode 100644 index 0000000000..584479410a --- /dev/null +++ b/internal/pkg/service/cli/cmd/remote/vault/delete/cmd.go @@ -0,0 +1,103 @@ +package deletevault + +import ( + "github.com/spf13/cobra" + + "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" + + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies" + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/helpmsg" + "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" + "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + saveManifest "github.com/keboola/keboola-as-code/pkg/lib/operation/project/local/manifest/save" +) + +type Flags struct { + StorageAPIHost configmap.Value[string] `configKey:"storage-api-host" configShorthand:"H" configUsage:"storage API host, eg. \"connection.keboola.com\""` + StorageAPIToken configmap.Value[string] `configKey:"storage-api-token" configShorthand:"t" configUsage:"storage API token from your project"` + Name configmap.Value[string] `configKey:"name" configUsage:"name of the vault variable to delete"` +} + +func DefaultFlags() Flags { + return Flags{} +} + +func Command(p dependencies.Provider) *cobra.Command { + cmd := &cobra.Command{ + Use: `delete [name]`, + Short: helpmsg.Read(`remote/vault/delete/short`), + Long: helpmsg.Read(`remote/vault/delete/long`), + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) (cmdErr error) { + f := Flags{} + if err := p.BaseScope().ConfigBinder().Bind(cmd.Context(), cmd.Flags(), args, &f); err != nil { + return err + } + + d, err := p.RemoteCommandScope(cmd.Context(), f.StorageAPIHost, f.StorageAPIToken) + if err != nil { + return err + } + + logger := d.Logger() + + prj, _, err := d.LocalProject(cmd.Context(), false) + if err != nil { + return err + } + + var variableName string + if len(args) > 0 { + variableName = args[0] + } else if f.Name.Value != "" { + variableName = f.Name.Value + } else { + variableName, _ = d.Dialogs().Ask(AskVariableName()) + } + + defer d.EventSender().SendCmdEvent(cmd.Context(), d.Clock().Now(), &cmdErr, "remote-vault-delete") + + logger.Infof(cmd.Context(), "Fetching vault variables...") + + allVariables, err := d.KeboolaProjectAPI().ListVariablesRequest(nil).Send(cmd.Context()) + if err != nil { + return errors.Errorf("failed to list vault variables: %w", err) + } + + var targetVariable *keboola.VaultVariable + for _, v := range *allVariables { + if v.Key == variableName { + targetVariable = v + break + } + } + + if targetVariable == nil { + return errors.Errorf("vault variable \"%s\" not found", variableName) + } + + logger.Infof(cmd.Context(), "Deleting vault variable \"%s\" (hash: %s)...", variableName, targetVariable.Hash) + + if _, err := d.KeboolaProjectAPI().DeleteVariableRequest(targetVariable.Hash).Send(cmd.Context()); err != nil { + return errors.Errorf("failed to delete vault variable: %w", err) + } + + logger.Infof(cmd.Context(), "Vault variable \"%s\" deleted successfully.", variableName) + + if prj.ProjectManifest().RemoveVaultVariable(targetVariable.Hash) { + if _, err := saveManifest.Run(cmd.Context(), prj.ProjectManifest(), prj.Fs(), d); err != nil { + return errors.Errorf("failed to save manifest: %w", err) + } + logger.Info(cmd.Context(), "Manifest updated successfully.") + } else { + logger.Info(cmd.Context(), "Variable was not in manifest.") + } + + return nil + }, + } + + configmap.MustGenerateFlags(cmd.Flags(), DefaultFlags()) + + return cmd +} diff --git a/internal/pkg/service/cli/cmd/remote/vault/delete/dialog.go b/internal/pkg/service/cli/cmd/remote/vault/delete/dialog.go new file mode 100644 index 0000000000..a3ef1f96f9 --- /dev/null +++ b/internal/pkg/service/cli/cmd/remote/vault/delete/dialog.go @@ -0,0 +1,13 @@ +package deletevault + +import ( + "github.com/keboola/keboola-as-code/internal/pkg/service/cli/prompt" +) + +func AskVariableName() *prompt.Question { + return &prompt.Question{ + Label: "Variable name", + Description: "Enter the vault variable name to delete.", + Validator: prompt.ValueRequired, + } +} diff --git a/internal/pkg/service/cli/cmd/sync/pull/cmd.go b/internal/pkg/service/cli/cmd/sync/pull/cmd.go index 57472cd558..7598953215 100644 --- a/internal/pkg/service/cli/cmd/sync/pull/cmd.go +++ b/internal/pkg/service/cli/cmd/sync/pull/cmd.go @@ -18,6 +18,7 @@ type Flags struct { Force configmap.Value[bool] `configKey:"force" configUsage:"ignore invalid local state"` DryRun configmap.Value[bool] `configKey:"dry-run" configUsage:"print what needs to be done"` CleanupRenameConflicts configmap.Value[bool] `configKey:"cleanup-rename-conflicts" configUsage:"enable cleanup mode for rename conflicts (removes conflicting destinations)"` + VaultEnabled configmap.Value[bool] `configKey:"vault-enabled" configUsage:"fetch and update vault variables in manifest"` } func DefaultFlags() Flags { @@ -69,6 +70,16 @@ func Command(p dependencies.Provider) *cobra.Command { return err } + if f.VaultEnabled.Value { + logger.Info(cmd.Context(), "Fetching vault variables...") + variables, err := d.KeboolaProjectAPI().ListVariablesRequest(nil).Send(cmd.Context()) + if err != nil { + return err + } + prj.ProjectManifest().SetVaultVariables(*variables) + logger.Infof(cmd.Context(), "Fetched %d vault variables", len(*variables)) + } + // Options options := pull.Options{ DryRun: f.DryRun.Value, diff --git a/internal/pkg/service/cli/helpmsg/msg/remote/vault/create/long.txt b/internal/pkg/service/cli/helpmsg/msg/remote/vault/create/long.txt new file mode 100644 index 0000000000..02692fd58f --- /dev/null +++ b/internal/pkg/service/cli/helpmsg/msg/remote/vault/create/long.txt @@ -0,0 +1,7 @@ +Create a new vault variable in the project. + +The command will interactively prompt for the variable name and value, +or you can provide the name as an argument. + +The variable will be created in Keboola Connection and automatically +added to the local manifest file. diff --git a/internal/pkg/service/cli/helpmsg/msg/remote/vault/create/short.txt b/internal/pkg/service/cli/helpmsg/msg/remote/vault/create/short.txt new file mode 100644 index 0000000000..28b9dc595e --- /dev/null +++ b/internal/pkg/service/cli/helpmsg/msg/remote/vault/create/short.txt @@ -0,0 +1 @@ +Create a new vault variable diff --git a/internal/pkg/service/cli/helpmsg/msg/remote/vault/delete/long.txt b/internal/pkg/service/cli/helpmsg/msg/remote/vault/delete/long.txt new file mode 100644 index 0000000000..220c20463e --- /dev/null +++ b/internal/pkg/service/cli/helpmsg/msg/remote/vault/delete/long.txt @@ -0,0 +1,7 @@ +Delete a vault variable from the project. + +The command will interactively prompt for the variable name, +or you can provide the name as an argument. + +The variable will be deleted from Keboola Connection and automatically +removed from the local manifest file if present. diff --git a/internal/pkg/service/cli/helpmsg/msg/remote/vault/delete/short.txt b/internal/pkg/service/cli/helpmsg/msg/remote/vault/delete/short.txt new file mode 100644 index 0000000000..ccc9ff023f --- /dev/null +++ b/internal/pkg/service/cli/helpmsg/msg/remote/vault/delete/short.txt @@ -0,0 +1 @@ +Delete a vault variable diff --git a/internal/pkg/service/cli/helpmsg/msg/remote/vault/long.txt b/internal/pkg/service/cli/helpmsg/msg/remote/vault/long.txt new file mode 100644 index 0000000000..a33c5d3d32 --- /dev/null +++ b/internal/pkg/service/cli/helpmsg/msg/remote/vault/long.txt @@ -0,0 +1,4 @@ +Manage vault variables in the project. + +Vault variables are secure key-value pairs stored in Keboola Connection +that can be used across your project configurations. diff --git a/internal/pkg/service/cli/helpmsg/msg/remote/vault/short.txt b/internal/pkg/service/cli/helpmsg/msg/remote/vault/short.txt new file mode 100644 index 0000000000..b3b759f7ba --- /dev/null +++ b/internal/pkg/service/cli/helpmsg/msg/remote/vault/short.txt @@ -0,0 +1 @@ +Manage vault variables in the project