diff --git a/go.mod b/go.mod index a1763aefbb..ac6dfa7a99 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/keboola/go-cloud-encrypt v0.0.0-20250422071622-41a5d5547c43 github.com/keboola/go-utils v1.4.0 - github.com/keboola/keboola-sdk-go/v2 v2.18.0 + github.com/keboola/keboola-sdk-go/v2 v2.17.1-0.20260326212557-482d079b522e github.com/keboola/keboola-sdk-go/v2/transfer v1.0.0 github.com/klauspost/compress v1.18.4 github.com/klauspost/pgzip v1.2.6 @@ -316,7 +316,7 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect - github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/hashicorp/hcl/v2 v2.23.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect diff --git a/go.sum b/go.sum index 186802765d..5dde4fb1fd 100644 --- a/go.sum +++ b/go.sum @@ -596,8 +596,8 @@ github.com/keboola/go-oauth2-proxy/v7 v7.13.1-0.20251120082210-251fbcb18c16 h1:m github.com/keboola/go-oauth2-proxy/v7 v7.13.1-0.20251120082210-251fbcb18c16/go.mod h1:2KeAM0/QPbyUAoky+PXVgQDt/5m0qNcn30z9jh4ig8A= github.com/keboola/go-utils v1.4.0 h1:WTyj95yrr8O8HxtC8TSTyUcElZiRGDeEdVvDpFo6HUo= github.com/keboola/go-utils v1.4.0/go.mod h1:IopwJzFz2gh0Yj3fUbIe2eamRoDKzbXvjqFjQyw3ZdQ= -github.com/keboola/keboola-sdk-go/v2 v2.18.0 h1:wkHTV4eUHr3aEwHmbo/Lhxq1uiYzdFbcMfY38Icvo2Q= -github.com/keboola/keboola-sdk-go/v2 v2.18.0/go.mod h1:dLDyVUt6gMPGoXgON8yKh6+k4VEMlByXMqkK7rL1kuE= +github.com/keboola/keboola-sdk-go/v2 v2.17.1-0.20260326212557-482d079b522e h1:J3AfxA2zcZU5aZt98fbDc7XKuqYi17ZqZh3sZISSZFI= +github.com/keboola/keboola-sdk-go/v2 v2.17.1-0.20260326212557-482d079b522e/go.mod h1:dLDyVUt6gMPGoXgON8yKh6+k4VEMlByXMqkK7rL1kuE= github.com/keboola/keboola-sdk-go/v2/transfer v1.0.0 h1:mjwUdS9W+0QDYycam20u0B3LSSEUpXFvHyH9Q5j6fwA= github.com/keboola/keboola-sdk-go/v2/transfer v1.0.0/go.mod h1:+K9kZRslskn0r3qZmyXXd7trrApNQrs1aVUcfbTm2V4= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= diff --git a/internal/pkg/fixtures/fixtures.go b/internal/pkg/fixtures/fixtures.go index aea74f51be..524f49962a 100644 --- a/internal/pkg/fixtures/fixtures.go +++ b/internal/pkg/fixtures/fixtures.go @@ -48,10 +48,10 @@ type Schedule struct { } type Sandbox struct { - Name string `json:"name" validate:"required"` - Type keboola.SandboxWorkspaceType `json:"type" validate:"required"` - Size string `json:"size,omitempty"` - UseKeyPair bool `json:"useKeyPair,omitempty"` // If true, create sandbox with key-pair authentication instead of password + Name string `json:"name" validate:"required"` + Type string `json:"type" validate:"required"` + Size string `json:"size,omitempty"` + UseKeyPair bool `json:"useKeyPair,omitempty"` // If true, create sandbox with key-pair authentication instead of password } type Bucket struct { @@ -173,7 +173,7 @@ func (r *ConfigRow) String() string { } func (s *Sandbox) String() string { - return s.Type.String() + "_" + s.Size + return s.Type + "_" + s.Size } func (b *Branch) ObjectName() string { diff --git a/internal/pkg/keboola/sandbox/sandbox.go b/internal/pkg/keboola/sandbox/sandbox.go deleted file mode 100644 index ff60d08559..0000000000 --- a/internal/pkg/keboola/sandbox/sandbox.go +++ /dev/null @@ -1,372 +0,0 @@ -// Package sandbox provides Python/R workspace types and operations that were removed -// from keboola-sdk-go v2.18.0. The SDK now exposes DataScienceApp for listing/fetching -// workspaces; this package bridges the gap for existing callers without requiring a -// full migration. -package sandbox - -import ( - "context" - "fmt" - "sync" - - "github.com/hashicorp/go-multierror" - "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" - "github.com/keboola/keboola-sdk-go/v2/pkg/request" - "golang.org/x/sync/errgroup" - - "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" -) - -// SandboxWorkspace represents a Python/R sandbox workspace. -// Host/User/Password are not populated when sourced from DataScienceApp; -// use StorageWorkspaceCreateCredentialsRequest to retrieve them when needed. -type SandboxWorkspace struct { - ID keboola.SandboxWorkspaceID - Type keboola.SandboxWorkspaceType - Size string - User string - Host string - URL string - Password string //nolint:gosec - Details *SandboxWorkspaceDetails - Credentials *SandboxWorkspaceCredentials -} - -// SandboxWorkspaceDetails holds connection details for a SQL workspace. -type SandboxWorkspaceDetails struct { - Connection struct { - Database string - Schema string - Warehouse string - } -} - -// SandboxWorkspaceCredentials contains BigQuery service-account credentials. -type SandboxWorkspaceCredentials struct { - Type string - ProjectID string - PrivateKeyID string - ClientEmail string - ClientID string - AuthURI string - TokenURI string - AuthProviderX509CertURL string - ClientX509CertURL string - PrivateKey string //nolint:gosec -} - -// SandboxWorkspaceWithConfig pairs a workspace instance with its keboola.sandboxes config. -type SandboxWorkspaceWithConfig struct { - SandboxWorkspace *SandboxWorkspace - Config *keboola.Config -} - -func (v SandboxWorkspaceWithConfig) String() string { - if keboola.SandboxWorkspaceSupportsSizes(v.SandboxWorkspace.Type) { - return fmt.Sprintf("ID: %s, Type: %s, Size: %s, Name: %s", - v.SandboxWorkspace.ID, v.SandboxWorkspace.Type, v.SandboxWorkspace.Size, v.Config.Name) - } - return fmt.Sprintf("ID: %s, Type: %s, Name: %s", - v.SandboxWorkspace.ID, v.SandboxWorkspace.Type, v.Config.Name) -} - -// createParams holds options for the keboola.sandboxes "create" queue job. -type createParams struct { - Type keboola.SandboxWorkspaceType - Shared bool - ExpireAfterHours uint64 - Size string - ImageVersion string - PublicKey string - LoginType string -} - -func (p createParams) toMap() map[string]any { - m := map[string]any{ - "task": "create", - "type": p.Type, - "shared": p.Shared, - "expirationAfterHours": p.ExpireAfterHours, - } - if len(p.Size) > 0 { - m["size"] = p.Size - } - if len(p.ImageVersion) > 0 { - m["imageVersion"] = p.ImageVersion - } - if len(p.PublicKey) > 0 { - m["publicKey"] = p.PublicKey - m["loginType"] = p.LoginType - } - return m -} - -// CreateSandboxWorkspaceOption configures workspace creation. -type CreateSandboxWorkspaceOption func(p *createParams) - -// WithSize sets the backend size (small, medium, large) for Python/R workspaces. -func WithSize(v string) CreateSandboxWorkspaceOption { - return func(p *createParams) { p.Size = v } -} - -// WithPublicKey configures keypair authentication with the given public key PEM. -// Note: loginType is set to "snowflake-person-keypair"; BigQuery workspaces use a -// different auth mechanism and should not use this option. -func WithPublicKey(v string) CreateSandboxWorkspaceOption { - return func(p *createParams) { - p.PublicKey = v - p.LoginType = "snowflake-person-keypair" - } -} - -// WithShared marks the workspace as shared. -func WithShared(v bool) CreateSandboxWorkspaceOption { - return func(p *createParams) { p.Shared = v } -} - -// WithExpireAfterHours sets automatic expiry. -func WithExpireAfterHours(v uint64) CreateSandboxWorkspaceOption { - return func(p *createParams) { p.ExpireAfterHours = v } -} - -// GetSandboxWorkspaceID reads the workspace instance ID from a keboola.sandboxes config. -// The ID is stored at parameters.id in the config content. -func GetSandboxWorkspaceID(c *keboola.Config) (keboola.SandboxWorkspaceID, error) { - id, found, err := c.Content.GetNested("parameters.id") - if err != nil { - return "", err - } - if !found { - return "", errors.Errorf("config is missing parameters.id") - } - out, ok := id.(string) - if !ok { - return "", errors.Errorf("config.parameters.id is not a string") - } - return keboola.SandboxWorkspaceID(out), nil -} - -// CreateSandboxWorkspace creates a keboola.sandboxes config, runs the creation queue job, -// waits for completion, then returns the workspace joined with its config. -func CreateSandboxWorkspace( - ctx context.Context, - api *keboola.AuthorizedAPI, - branchID keboola.BranchID, - name string, - wsType keboola.SandboxWorkspaceType, - opts ...CreateSandboxWorkspaceOption, -) (*SandboxWorkspaceWithConfig, error) { - emptyConfig, err := api.CreateSandboxWorkspaceConfigRequest(branchID, name).Send(ctx) - if err != nil { - return nil, err - } - - p := createParams{Type: wsType} - for _, opt := range opts { - opt(&p) - } - - req := api.NewCreateJobRequest(keboola.SandboxWorkspacesComponent). - WithConfig(emptyConfig.ID). - WithConfigData(map[string]any{"parameters": p.toMap()}). - Build(). - WithOnSuccess(func(ctx context.Context, result *keboola.QueueJob) error { - return api.WaitForQueueJob(ctx, result.ID) - }) - if _, err = request.NewAPIRequest(request.NoResult{}, req).Send(ctx); err != nil { - return nil, err - } - - return GetSandboxWorkspace(ctx, api, branchID, emptyConfig.ID) -} - -// DeleteSandboxWorkspace deletes the workspace instance via queue job, then deletes the config. -func DeleteSandboxWorkspace( - ctx context.Context, - api *keboola.AuthorizedAPI, - branchID keboola.BranchID, - configID keboola.ConfigID, - workspaceID keboola.SandboxWorkspaceID, -) error { - if _, err := api.DeleteSandboxWorkspaceJobRequest(workspaceID).Send(ctx); err != nil { - return err - } - _, err := api.DeleteSandboxWorkspaceConfigRequest(branchID, configID).Send(ctx) - return err -} - -// GetSandboxWorkspace fetches the config then the DataScienceApp, returning both joined. -func GetSandboxWorkspace( - ctx context.Context, - api *keboola.AuthorizedAPI, - branchID keboola.BranchID, - configID keboola.ConfigID, -) (*SandboxWorkspaceWithConfig, error) { - config, err := api.GetSandboxWorkspaceConfigRequest(branchID, configID).Send(ctx) - if err != nil { - return nil, err - } - - workspaceID, err := GetSandboxWorkspaceID(config) - if err != nil { - return nil, err - } - - app, err := api.GetDataScienceAppRequest(keboola.DataScienceAppID(workspaceID)).Send(ctx) - if err != nil { - return nil, err - } - - return &SandboxWorkspaceWithConfig{ - SandboxWorkspace: dataScienceAppToWorkspace(app), - Config: config, - }, nil -} - -// ListSandboxWorkspaces fetches Python/R workspaces for a branch in parallel with configs, -// joining by DataScienceApp.ConfigID. It also returns all sandbox configs so callers -// can look up SQL workspace names without a second API call. -func ListSandboxWorkspaces( - ctx context.Context, - api *keboola.AuthorizedAPI, - branchID keboola.BranchID, -) ([]*SandboxWorkspaceWithConfig, []*keboola.Config, error) { - var configs []*keboola.Config - var apps []*keboola.DataScienceApp - wg := &sync.WaitGroup{} - mu := &sync.Mutex{} - var err error - - wg.Go(func() { - data, e := api.ListSandboxWorkspaceConfigRequest(branchID).Send(ctx) - if e != nil { - mu.Lock() - defer mu.Unlock() - err = multierror.Append(err, e) - return - } - configs = *data - }) - - wg.Go(func() { - data, e := api.ListDataScienceAppsRequest( - keboola.WithDataScienceAppsComponentID(keboola.ComponentID(keboola.SandboxWorkspacesComponent)), - keboola.WithDataScienceAppsBranchID(branchID.String()), - keboola.WithDataScienceAppsType(keboola.DataScienceAppTypePython, keboola.DataScienceAppTypeR), - ).Send(ctx) - if e != nil { - mu.Lock() - defer mu.Unlock() - err = multierror.Append(err, e) - return - } - apps = *data - }) - - wg.Wait() - if err != nil { - return nil, nil, err - } - - appsByConfigID := make(map[string]*keboola.DataScienceApp, len(apps)) - for _, app := range apps { - appsByConfigID[app.ConfigID] = app - } - - out := make([]*SandboxWorkspaceWithConfig, 0) - for _, config := range configs { - app, found := appsByConfigID[config.ID.String()] - if !found { - continue - } - out = append(out, &SandboxWorkspaceWithConfig{ - SandboxWorkspace: dataScienceAppToWorkspace(app), - Config: config, - }) - } - return out, configs, nil -} - -// ListAllWorkspaces fetches Python/R workspaces and SQL editor sessions in parallel, -// returning a combined list and the raw sessions (needed by callers that look up credentials). -func ListAllWorkspaces( - ctx context.Context, - api *keboola.AuthorizedAPI, - branchID keboola.BranchID, -) ([]*SandboxWorkspaceWithConfig, []*keboola.EditorSession, error) { - var pyRWorkspaces []*SandboxWorkspaceWithConfig - var allConfigs []*keboola.Config - var sessions []*keboola.EditorSession - - grp, grpCtx := errgroup.WithContext(ctx) - grp.Go(func() error { - var e error - pyRWorkspaces, allConfigs, e = ListSandboxWorkspaces(grpCtx, api, branchID) - return e - }) - grp.Go(func() error { - result, e := api.ListEditorSessionsRequest().Send(grpCtx) - if e != nil { - return e - } - sessions = *result - return nil - }) - if err := grp.Wait(); err != nil { - return nil, nil, err - } - - configNameMap := make(map[string]string) - for _, c := range allConfigs { - configNameMap[c.ID.String()] = c.Name - } - - all := make([]*SandboxWorkspaceWithConfig, 0, len(pyRWorkspaces)+len(sessions)) - all = append(all, pyRWorkspaces...) - for _, s := range sessions { - name := configNameMap[s.ConfigurationID] - all = append(all, &SandboxWorkspaceWithConfig{ - Config: &keboola.Config{ - ConfigKey: keboola.ConfigKey{ID: keboola.ConfigID(s.ConfigurationID)}, - Name: name, - }, - SandboxWorkspace: &SandboxWorkspace{ - ID: keboola.SandboxWorkspaceID(s.ID), - Type: keboola.SandboxWorkspaceType(s.BackendType), - }, - }) - } - - return all, sessions, nil -} - -// WorkspaceFromStorage constructs a SandboxWorkspace from StorageWorkspace credentials, -// used when credentials come from an editor session (SQL workspaces). -func WorkspaceFromStorage(sw *keboola.StorageWorkspace, wsType keboola.SandboxWorkspaceType) *SandboxWorkspace { - deref := func(s *string) string { - if s == nil { - return "" - } - return *s - } - details := &SandboxWorkspaceDetails{} - details.Connection.Database = deref(sw.StorageWorkspaceDetails.Database) - details.Connection.Schema = deref(sw.StorageWorkspaceDetails.Schema) - details.Connection.Warehouse = deref(sw.StorageWorkspaceDetails.Warehouse) - return &SandboxWorkspace{ - Type: wsType, - Host: deref(sw.StorageWorkspaceDetails.Host), - User: deref(sw.StorageWorkspaceDetails.User), - Details: details, - } -} - -func dataScienceAppToWorkspace(app *keboola.DataScienceApp) *SandboxWorkspace { - return &SandboxWorkspace{ - ID: keboola.SandboxWorkspaceID(app.ID), - Type: keboola.SandboxWorkspaceType(app.Type), - Size: app.Size, - URL: app.URL, - Details: &SandboxWorkspaceDetails{}, // always non-nil; fields are empty for Python/R - } -} diff --git a/internal/pkg/service/cli/cmd/dbt/dbtinit/cmd.go b/internal/pkg/service/cli/cmd/dbt/dbtinit/cmd.go index 27c13893b5..860bd0e0aa 100644 --- a/internal/pkg/service/cli/cmd/dbt/dbtinit/cmd.go +++ b/internal/pkg/service/cli/cmd/dbt/dbtinit/cmd.go @@ -1,6 +1,8 @@ package dbtinit import ( + "strings" + "github.com/spf13/cobra" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies" @@ -57,6 +59,7 @@ func Command(p dependencies.Provider) *cobra.Command { if err != nil { return err } + opts.BaseURL = baseURLFromHost(d.StorageAPIHost()) // Send cmd successful/failed event defer d.EventSender().SendCmdEvent(cmd.Context(), d.Clock().Now(), &cmdErr, "dbt-init") @@ -69,3 +72,10 @@ func Command(p dependencies.Provider) *cobra.Command { return cmd } + +// baseURLFromHost derives the Keboola Query Service URL from the Storage API host. +// "https://connection.keboola.com" → "https://query.keboola.com" +func baseURLFromHost(host string) string { + bare := strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://") + return "https://query." + strings.TrimPrefix(bare, "connection.") +} diff --git a/internal/pkg/service/cli/cmd/dbt/generate/env/cmd.go b/internal/pkg/service/cli/cmd/dbt/generate/env/cmd.go index 545a2ed62f..9db52cf3fb 100644 --- a/internal/pkg/service/cli/cmd/dbt/generate/env/cmd.go +++ b/internal/pkg/service/cli/cmd/dbt/generate/env/cmd.go @@ -1,14 +1,19 @@ package env import ( + "strings" + + "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/helpmsg" "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" - "github.com/keboola/keboola-as-code/pkg/lib/operation/dbt/generate/env" + genenv "github.com/keboola/keboola-as-code/pkg/lib/operation/dbt/generate/env" + wsinfo "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" + listOp "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace/list" ) type Flags struct { @@ -51,20 +56,62 @@ func Command(p dependencies.Provider) *cobra.Command { return errors.Errorf("cannot get default branch: %w", err) } - allWorkspaces, sessions, err := sandbox.ListAllWorkspaces(cmd.Context(), d.KeboolaProjectAPI(), branch.ID) - if err != nil { + // Fetch Python/R workspaces and editor sessions in parallel. + var pyRWorkspaces []*wsinfo.WorkspaceWithConfig + var allConfigs []*keboola.Config + var sessions []*keboola.EditorSession + + grp, grpCtx := errgroup.WithContext(cmd.Context()) + grp.Go(func() error { + var e error + pyRWorkspaces, allConfigs, e = listOp.ListPyRWorkspaces(grpCtx, d.KeboolaProjectAPI(), branch.ID) + return e + }) + grp.Go(func() error { + result, e := d.KeboolaProjectAPI().ListEditorSessionsRequest().Send(grpCtx) + if e != nil { + return e + } + sessions = *result + return nil + }) + if err := grp.Wait(); err != nil { return err } - opts, err := AskGenerateEnv(cmd.Context(), branch.BranchKey, branch.ID, d.Dialogs(), allWorkspaces, sessions, f, p.BaseScope().Environment(), d.KeboolaProjectAPI()) + // Build config name map for editor session name lookup. + configNameMap := make(map[string]string) + for _, c := range allConfigs { + configNameMap[c.ID.String()] = c.Name + } + + // Build combined workspace list: Python/R + SQL editor sessions. + allWorkspaces := make([]*wsinfo.WorkspaceWithConfig, 0, len(pyRWorkspaces)+len(sessions)) + allWorkspaces = append(allWorkspaces, pyRWorkspaces...) + for _, s := range sessions { + name := configNameMap[s.ConfigurationID] + session := s + allWorkspaces = append(allWorkspaces, &wsinfo.WorkspaceWithConfig{ + Config: &keboola.Config{ + ConfigKey: keboola.ConfigKey{ID: keboola.ConfigID(s.ConfigurationID)}, + Name: name, + }, + Session: session, + }) + } + + opts, err := AskGenerateEnv(cmd.Context(), branch.BranchKey, branch.ID, d.Dialogs(), allWorkspaces, f, p.BaseScope().Environment(), d.KeboolaProjectAPI()) if err != nil { return err } + // Set BaseURL for keboola adapter vars (only meaningful for SQL workspaces with an editor session). + opts.Workspace.BaseURL = baseURLFromHost(d.StorageAPIHost()) + // Send cmd successful/failed event defer d.EventSender().SendCmdEvent(cmd.Context(), d.Clock().Now(), &cmdErr, "dbt-generate-env") - return env.Run(cmd.Context(), opts, d) + return genenv.Run(cmd.Context(), opts, d) }, } @@ -72,3 +119,12 @@ func Command(p dependencies.Provider) *cobra.Command { return cmd } + +// baseURLFromHost derives the Keboola Query Service URL from the Storage API host. +// "https://connection.keboola.com" → "https://query.keboola.com" +func baseURLFromHost(host string) string { + // host is already normalized with https:// prefix by the dependencies layer. + // Strip protocol, replace "connection." prefix with "query.", re-add https://. + bare := strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://") + return "https://query." + strings.TrimPrefix(bare, "connection.") +} diff --git a/internal/pkg/service/cli/cmd/dbt/generate/env/dialog.go b/internal/pkg/service/cli/cmd/dbt/generate/env/dialog.go index fa4839b3b2..1c44ac2a8c 100644 --- a/internal/pkg/service/cli/cmd/dbt/generate/env/dialog.go +++ b/internal/pkg/service/cli/cmd/dbt/generate/env/dialog.go @@ -9,10 +9,10 @@ import ( "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" kenv "github.com/keboola/keboola-as-code/internal/pkg/env" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/dialog" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" genenv "github.com/keboola/keboola-as-code/pkg/lib/operation/dbt/generate/env" + wsinfo "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" ) func AskGenerateEnv( @@ -20,8 +20,7 @@ func AskGenerateEnv( branchKey keboola.BranchKey, branchID keboola.BranchID, d *dialog.Dialogs, - allWorkspaces []*sandbox.SandboxWorkspaceWithConfig, - sessions []*keboola.EditorSession, + allWorkspaces []*wsinfo.WorkspaceWithConfig, f Flags, envs kenv.Provider, api *keboola.AuthorizedAPI, @@ -42,32 +41,26 @@ func AskGenerateEnv( normalizedName := strings.ToUpper(strings.NewReplacer(" ", "_", "-", "_").Replace(workspaceName)) privateKeyEnvVar := fmt.Sprintf("TEST_SANDBOX_%s_PRIVATE_KEY", normalizedName) privateKey := envs.Get(privateKeyEnvVar) - useKeyPair := len(privateKey) > 0 - if keboola.SandboxWorkspaceSupportsSizes(workspace.SandboxWorkspace.Type) { - // Python/R workspace — use credentials directly. + if workspace.App != nil { + // Python/R workspace — credential fields are not available from the listing. return genenv.Options{ BranchKey: branchKey, TargetName: targetName, - Workspace: workspace.SandboxWorkspace, // already *sandbox.SandboxWorkspace + Workspace: genenv.WorkspaceDetails{ + Type: workspace.Type(), + }, UseKeyPair: useKeyPair, PrivateKey: privateKey, }, nil } // SQL workspace (Snowflake/BigQuery) — fetch StorageWorkspace credentials via the editor session. - // Find the editor session for this workspace by matching ConfigurationID. - var matchedSession *keboola.EditorSession - for _, s := range sessions { - if s.ConfigurationID == workspace.Config.ID.String() { - matchedSession = s - break - } - } - if matchedSession == nil { + if workspace.Session == nil { return genenv.Options{}, errors.Errorf(`no active editor session found for workspace %q`, workspace.Config.Name) } + matchedSession := workspace.Session workspaceIDUint, err := strconv.ParseUint(matchedSession.WorkspaceID, 10, 64) if err != nil { @@ -81,7 +74,23 @@ func AskGenerateEnv( return genenv.Options{}, errors.Errorf("cannot fetch workspace credentials: %w", err) } - sandboxWS := sandbox.WorkspaceFromStorage(storageWS, keboola.SandboxWorkspaceType(matchedSession.BackendType)) + deref := func(s *string) string { + if s == nil { + return "" + } + return *s + } + + ws := genenv.WorkspaceDetails{ + Type: string(matchedSession.BackendType), + Host: deref(storageWS.StorageWorkspaceDetails.Host), + User: deref(storageWS.StorageWorkspaceDetails.User), + Database: deref(storageWS.StorageWorkspaceDetails.Database), + Schema: deref(storageWS.StorageWorkspaceDetails.Schema), + Warehouse: deref(storageWS.StorageWorkspaceDetails.Warehouse), + BranchID: branchID, + WorkspaceID: matchedSession.WorkspaceID, + } // Use server-provided private key for SQL workspaces when available. if len(privateKey) == 0 && storageWS.StorageWorkspaceDetails.PrivateKey != nil && len(*storageWS.StorageWorkspaceDetails.PrivateKey) > 0 { @@ -92,7 +101,7 @@ func AskGenerateEnv( return genenv.Options{ BranchKey: branchKey, TargetName: targetName, - Workspace: sandboxWS, + Workspace: ws, UseKeyPair: useKeyPair, PrivateKey: privateKey, }, nil diff --git a/internal/pkg/service/cli/cmd/remote/workspace/create/dialog.go b/internal/pkg/service/cli/cmd/remote/workspace/create/dialog.go index 5d67450840..4e43f400cc 100644 --- a/internal/pkg/service/cli/cmd/remote/workspace/create/dialog.go +++ b/internal/pkg/service/cli/cmd/remote/workspace/create/dialog.go @@ -1,27 +1,14 @@ package create import ( - "strings" - - "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" - "github.com/keboola/keboola-as-code/internal/pkg/service/cli/dialog" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/prompt" "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace/create" ) -// sandboxWorkspaceTypesToStrings converts a slice of SandboxWorkspaceType to []string. -// This helper function is needed because the SDK returns custom types that need to be converted to strings. -func sandboxWorkspaceTypesToStrings(types []keboola.SandboxWorkspaceType) []string { - result := make([]string, len(types)) - for i, typ := range types { - result[i] = string(typ) - } - return result -} - func AskCreateWorkspace(d *dialog.Dialogs, f Flags) (create.CreateOptions, error) { opts := create.CreateOptions{} @@ -37,7 +24,7 @@ func AskCreateWorkspace(d *dialog.Dialogs, f Flags) (create.CreateOptions, error } opts.Type = typ - if keboola.SandboxWorkspaceSupportsSizes(typ) { + if workspace.WorkspaceSupportsSizes(typ) { size, err := askWorkspaceSize(d, f.Size) if err != nil { return opts, err @@ -63,37 +50,38 @@ func askWorkspaceName(d *dialog.Dialogs, workspaceName configmap.Value[string]) } } -func askWorkspaceType(d *dialog.Dialogs, workspaceType configmap.Value[string]) (keboola.SandboxWorkspaceType, error) { +func askWorkspaceType(d *dialog.Dialogs, workspaceType configmap.Value[string]) (workspace.WorkspaceType, error) { if workspaceType.IsSet() { typ := workspaceType.Value - // Convert the string to SandboxWorkspaceType for map lookup - if !keboola.SandboxWorkspaceTypesMap()[keboola.SandboxWorkspaceType(typ)] { - return "", errors.Errorf("invalid workspace type, must be one of: %s", strings.Join(sandboxWorkspaceTypesToStrings(keboola.SandboxWorkspaceTypesOrdered()), ", ")) + if !workspace.WorkspaceTypesMap()[typ] { + return "", errors.Errorf("invalid workspace type, must be one of: %s", + formatList(workspace.WorkspaceTypesOrdered())) } - return keboola.SandboxWorkspaceType(typ), nil + return typ, nil } else { v, ok := d.Select(&prompt.Select{ Label: "Select a type for the new workspace", - Options: sandboxWorkspaceTypesToStrings(keboola.SandboxWorkspaceTypesOrdered()), + Options: workspace.WorkspaceTypesOrdered(), }) if !ok { return "", errors.New("missing workspace type, please specify it") } - return keboola.SandboxWorkspaceType(v), nil + return v, nil } } func askWorkspaceSize(d *dialog.Dialogs, workspaceSize configmap.Value[string]) (string, error) { if workspaceSize.IsSet() { size := workspaceSize.Value - if !keboola.SandboxWorkspaceSizesMap()[size] { - return "", errors.Errorf("invalid workspace size, must be one of: %s", strings.Join(keboola.SandboxWorkspaceSizesOrdered(), ", ")) + if !workspace.WorkspaceSizesMap()[size] { + return "", errors.Errorf("invalid workspace size, must be one of: %s", + formatList(workspace.WorkspaceSizesOrdered())) } return size, nil } else { v, ok := d.Select(&prompt.Select{ Label: "Select a size for the new workspace", - Options: keboola.SandboxWorkspaceSizesOrdered(), + Options: workspace.WorkspaceSizesOrdered(), }) if !ok { return "", errors.New("missing workspace size, please specify it") @@ -101,3 +89,14 @@ func askWorkspaceSize(d *dialog.Dialogs, workspaceSize configmap.Value[string]) return v, nil } } + +func formatList(items []string) string { + result := "" + for i, item := range items { + if i > 0 { + result += ", " + } + result += item + } + return result +} diff --git a/internal/pkg/service/cli/cmd/remote/workspace/delete/cmd.go b/internal/pkg/service/cli/cmd/remote/workspace/delete/cmd.go index 6f422ec67d..bf4b4fbe7d 100644 --- a/internal/pkg/service/cli/cmd/remote/workspace/delete/cmd.go +++ b/internal/pkg/service/cli/cmd/remote/workspace/delete/cmd.go @@ -1,14 +1,17 @@ package deleteworkspace import ( + "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/helpmsg" "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + wsinfo "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" deleteOp "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace/delete" + listOp "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace/list" ) type Flags struct { @@ -45,11 +48,50 @@ func Command(p dependencies.Provider) *cobra.Command { return errors.Errorf("cannot get default branch: %w", err) } - allWorkspaces, _, err := sandbox.ListAllWorkspaces(cmd.Context(), d.KeboolaProjectAPI(), branch.ID) - if err != nil { + // Fetch Python/R workspaces and editor sessions in parallel. + var pyRWorkspaces []*wsinfo.WorkspaceWithConfig + var allConfigs []*keboola.Config + var sessions []*keboola.EditorSession + + grp, grpCtx := errgroup.WithContext(cmd.Context()) + grp.Go(func() error { + var e error + pyRWorkspaces, allConfigs, e = listOp.ListPyRWorkspaces(grpCtx, d.KeboolaProjectAPI(), branch.ID) + return e + }) + grp.Go(func() error { + result, e := d.KeboolaProjectAPI().ListEditorSessionsRequest().Send(grpCtx) + if e != nil { + return e + } + sessions = *result + return nil + }) + if err := grp.Wait(); err != nil { return err } + // Build config name map for editor session name lookup. + configNameMap := make(map[string]string) + for _, c := range allConfigs { + configNameMap[c.ID.String()] = c.Name + } + + // Build combined list: Python/R workspaces + SQL editor sessions. + allWorkspaces := make([]*wsinfo.WorkspaceWithConfig, 0, len(pyRWorkspaces)+len(sessions)) + allWorkspaces = append(allWorkspaces, pyRWorkspaces...) + for _, s := range sessions { + name := configNameMap[s.ConfigurationID] + allWorkspaces = append(allWorkspaces, &wsinfo.WorkspaceWithConfig{ + Config: &keboola.Config{ + ConfigKey: keboola.ConfigKey{ID: keboola.ConfigID(s.ConfigurationID)}, + Name: name, + }, + Session: s, + }) + } + + ws, err := d.Dialogs().AskWorkspace(allWorkspaces, f.WorkspaceID) if err != nil { return err diff --git a/internal/pkg/service/cli/dialog/workspaces.go b/internal/pkg/service/cli/dialog/workspaces.go index be45f2971d..865766d52a 100644 --- a/internal/pkg/service/cli/dialog/workspaces.go +++ b/internal/pkg/service/cli/dialog/workspaces.go @@ -3,13 +3,13 @@ package dialog import ( "fmt" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/service/cli/prompt" "github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + wsinfo "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" ) -func (p *Dialogs) AskWorkspace(allWorkspaces []*sandbox.SandboxWorkspaceWithConfig, id configmap.Value[string]) (*sandbox.SandboxWorkspaceWithConfig, error) { +func (p *Dialogs) AskWorkspace(allWorkspaces []*wsinfo.WorkspaceWithConfig, id configmap.Value[string]) (*wsinfo.WorkspaceWithConfig, error) { if id.IsSet() { workspaceID := id.Value for _, w := range allWorkspaces { diff --git a/internal/pkg/utils/testproject/project.go b/internal/pkg/utils/testproject/project.go index c1c5fea0df..6f4bfee76d 100644 --- a/internal/pkg/utils/testproject/project.go +++ b/internal/pkg/utils/testproject/project.go @@ -25,11 +25,10 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/env" "github.com/keboola/keboola-as-code/internal/pkg/filesystem" "github.com/keboola/keboola-as-code/internal/pkg/fixtures" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" - "github.com/keboola/keboola-as-code/internal/pkg/utils/crypto" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" "github.com/keboola/keboola-as-code/internal/pkg/utils/testhelper" "github.com/keboola/keboola-as-code/internal/pkg/utils/ulid" + wsinfo "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" ) type Project struct { @@ -450,7 +449,7 @@ func (p *Project) createSandboxes(defaultBranchID keboola.BranchID, sandboxes [] for _, fixture := range sandboxes { wg.Go(func() { p.logf("▶ Sandbox \"%s\"...", fixture.Name) - if keboola.SandboxWorkspaceSupportsSizes(fixture.Type) { + if wsinfo.WorkspaceSupportsSizes(fixture.Type) { p.createPythonRSandbox(ctx, defaultBranchID, fixture, errs) } else { p.createSQLEditorSession(ctx, defaultBranchID, fixture, errs) @@ -467,32 +466,26 @@ func (p *Project) createSandboxes(defaultBranchID keboola.BranchID, sandboxes [] } func (p *Project) createPythonRSandbox(ctx context.Context, branchID keboola.BranchID, fixture *fixtures.Sandbox, errs errors.MultiError) { - opts := make([]sandbox.CreateSandboxWorkspaceOption, 0) - if len(fixture.Size) > 0 { - opts = append(opts, sandbox.WithSize(fixture.Size)) - } - - var privateKeyPEM string - if fixture.UseKeyPair { - var publicKeyPEM string - var err error - if privateKeyPEM, publicKeyPEM, err = crypto.GenerateRSAKeyPairPEM(); err != nil { - errs.Append(errors.Errorf("could not generate key-pair for sandbox \"%s\": %w", fixture.Name, err)) - return - } - opts = append(opts, sandbox.WithPublicKey(publicKeyPEM)) + config, err := p.keboolaProjectAPI.CreateSandboxWorkspaceConfigRequest(branchID, fixture.Name).Send(ctx) + if err != nil { + errs.Append(errors.Errorf("could not create sandbox config \"%s\": %w", fixture.Name, err)) + return } - ws, err := sandbox.CreateSandboxWorkspace(ctx, p.keboolaProjectAPI, branchID, fixture.Name, fixture.Type, opts...) + _, err = p.keboolaProjectAPI.CreateDataScienceSandboxRequest(keboola.CreateDataScienceSandboxPayload{ + Type: keboola.DataScienceAppType(fixture.Type), + ConfigurationID: string(config.ID), + ComponentID: string(keboola.SandboxWorkspacesComponent), + BranchID: branchID.String(), + Size: keboola.DataScienceSandboxSize(fixture.Size), + }).Send(ctx) if err != nil { errs.Append(errors.Errorf("could not create sandbox \"%s\": %w", fixture.Name, err)) return } - p.logf("✔️ Sandbox \"%s\"(%s).", ws.Config.Name, ws.Config.ID) - p.setEnv(fmt.Sprintf("TEST_SANDBOX_%s_ID", fixture.Name), ws.Config.ID.String()) - if len(privateKeyPEM) > 0 { - p.setEnv(fmt.Sprintf("TEST_SANDBOX_%s_PRIVATE_KEY", fixture.Name), privateKeyPEM) - } + + p.logf("✔️ Sandbox \"%s\"(%s).", fixture.Name, config.ID) + p.setEnv(fmt.Sprintf("TEST_SANDBOX_%s_ID", fixture.Name), config.ID.String()) } func (p *Project) createSQLEditorSession(ctx context.Context, branchID keboola.BranchID, fixture *fixtures.Sandbox, errs errors.MultiError) { diff --git a/internal/pkg/utils/testproject/snapshot.go b/internal/pkg/utils/testproject/snapshot.go index d0137abc49..7563a0fa30 100644 --- a/internal/pkg/utils/testproject/snapshot.go +++ b/internal/pkg/utils/testproject/snapshot.go @@ -14,9 +14,9 @@ import ( "golang.org/x/sync/errgroup" "github.com/keboola/keboola-as-code/internal/pkg/fixtures" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" "github.com/keboola/keboola-as-code/internal/pkg/utils/reflecthelper" + wslist "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace/list" ) // NewSnapshot - to validate final project state in tests. @@ -139,18 +139,20 @@ func (p *Project) NewSnapshot() (*fixtures.ProjectSnapshot, error) { return req.SendOrErr(ctx) }) - workspacesMap := make(map[string]*sandbox.SandboxWorkspace) + workspacesMap := make(map[string]*keboola.DataScienceApp) grp.Go(func() error { defaultBranch, err := p.DefaultBranch() if err != nil { return err } - workspaces, _, err := sandbox.ListSandboxWorkspaces(ctx, p.keboolaProjectAPI, defaultBranch.ID) + workspaces, _, err := wslist.ListPyRWorkspaces(ctx, p.keboolaProjectAPI, defaultBranch.ID) if err != nil { return err } for _, w := range workspaces { - workspacesMap[w.SandboxWorkspace.ID.String()] = w.SandboxWorkspace + if w.App != nil { + workspacesMap[string(w.App.ID)] = w.App + } } return nil }) @@ -300,26 +302,10 @@ func (p *Project) NewSnapshot() (*fixtures.ProjectSnapshot, error) { // Join sandbox instances with config name for key, config := range configsMap { - if config.ComponentID == keboola.SandboxWorkspacesComponent { - // SQL (Snowflake/BigQuery): check editor sessions first - if session, found := editorSessionsMap[key.ID.String()]; found { - snapshot.Sandboxes = append(snapshot.Sandboxes, &fixtures.Sandbox{Name: config.Name, Type: keboola.SandboxWorkspaceType(session.BackendType)}) - continue - } - - // Python/R: fall back to sandbox workspace instance lookup - sandboxID, err := sandbox.GetSandboxWorkspaceID(config.ToAPI().Config) - if err != nil { - snapshot.Sandboxes = append(snapshot.Sandboxes, &fixtures.Sandbox{Name: "SANDBOX INSTANCE ID NOT SET"}) - continue - } - - if sandbox, found := workspacesMap[sandboxID.String()]; found { - snapshot.Sandboxes = append(snapshot.Sandboxes, &fixtures.Sandbox{Name: config.Name, Type: sandbox.Type, Size: sandbox.Size}) - } else { - snapshot.Sandboxes = append(snapshot.Sandboxes, &fixtures.Sandbox{Name: "SANDBOX INSTANCE NOT FOUND"}) - } + if config.ComponentID != keboola.SandboxWorkspacesComponent { + continue } + snapshot.Sandboxes = append(snapshot.Sandboxes, sandboxFromConfig(key, config, editorSessionsMap, workspacesMap)) } // Sort by name @@ -335,6 +321,35 @@ func (p *Project) NewSnapshot() (*fixtures.ProjectSnapshot, error) { return snapshot, nil } +// sandboxFromConfig converts a keboola.sandboxes config to a fixtures.Sandbox snapshot entry. +// SQL (Snowflake/BigQuery) workspaces are identified by their editor session; Python/R by parameters.id. +func sandboxFromConfig( + key keboola.ConfigKey, + config *fixtures.Config, + editorSessionsMap map[string]*keboola.EditorSession, + workspacesMap map[string]*keboola.DataScienceApp, +) *fixtures.Sandbox { + // SQL: check editor sessions first. + if session, found := editorSessionsMap[key.ID.String()]; found { + return &fixtures.Sandbox{Name: config.Name, Type: string(session.BackendType)} + } + + // Python/R: look up DataScienceApp via parameters.id stored in config content. + wsID, found, err := config.ToAPI().Content.GetNested("parameters.id") + if err != nil || !found { + return &fixtures.Sandbox{Name: "SANDBOX INSTANCE ID NOT SET"} + } + wsIDStr, ok := wsID.(string) + if !ok { + return &fixtures.Sandbox{Name: "SANDBOX INSTANCE ID NOT SET"} + } + + if app, found := workspacesMap[wsIDStr]; found { + return &fixtures.Sandbox{Name: config.Name, Type: string(app.Type), Size: app.Size} + } + return &fixtures.Sandbox{Name: "SANDBOX INSTANCE NOT FOUND"} +} + func normalizeChangeDesc(str string) string { // Default description if object has been created by test if str == "created by test" { diff --git a/pkg/lib/operation/dbt/generate/env/operation.go b/pkg/lib/operation/dbt/generate/env/operation.go index 4dc9abbd8a..ab6ca86487 100644 --- a/pkg/lib/operation/dbt/generate/env/operation.go +++ b/pkg/lib/operation/dbt/generate/env/operation.go @@ -10,17 +10,32 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/dbt" "github.com/keboola/keboola-as-code/internal/pkg/filesystem" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" "github.com/keboola/keboola-as-code/pkg/lib/operation/dbt/listbuckets" ) +// WorkspaceDetails holds the connection details for a workspace used in dbt env generation. +type WorkspaceDetails struct { + Type string // workspace type string, e.g. "snowflake", "python" + Host string + User string + Password string //nolint:gosec + Database string + Schema string + Warehouse string + // Fields for the keboola_snowflake dbt adapter (populated for SQL workspaces). + // When BaseURL is set, DBT_KBC_{TARGET}_BASE_URL / _BRANCH_ID / _WORKSPACE_ID are written. + BaseURL string // e.g. "https://query.keboola.com" + BranchID keboola.BranchID + WorkspaceID string // numeric workspace ID from EditorSession +} + type Options struct { BranchKey keboola.BranchKey TargetName string - Workspace *sandbox.SandboxWorkspace + Workspace WorkspaceDetails PrivateKey string //nolint:gosec UseKeyPair bool // Whether key-pair authentication was requested (only add private key if true) Buckets []listbuckets.Bucket // optional, set if the buckets have been loaded in a parent command @@ -31,21 +46,18 @@ type dependencies interface { LocalDbtProject(ctx context.Context) (*dbt.Project, bool, error) Logger() log.Logger Telemetry() telemetry.Telemetry - Fs() filesystem.Fs // Add filesystem dependency + Fs() filesystem.Fs } func Run(ctx context.Context, o Options, d dependencies) (err error) { ctx, span := d.Telemetry().Tracer().Start(ctx, "keboola.go.operation.dbt.generate.env") defer span.End(&err) - // Check that we are in dbt directory - // Get dbt project dbtProject, _, err := d.LocalDbtProject(ctx) if err != nil { return err } - // List bucket, if not set o.Buckets, err = listbuckets.Run(ctx, listbuckets.Options{BranchKey: o.BranchKey, TargetName: o.TargetName}, d) if err != nil { return errors.Errorf("could not list buckets: %w", err) @@ -53,25 +65,23 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) { targetUpper := strings.ToUpper(o.TargetName) host := o.Workspace.Host - if o.Workspace.Type == keboola.SandboxWorkspaceTypeSnowflake { + if o.Workspace.Type == "snowflake" { host = strings.Replace(host, ".snowflakecomputing.com", "", 1) } - // Prepare content for .env.local - var envContent strings.Builder envVars := make(map[string]string) - envVars[fmt.Sprintf("DBT_KBC_%s_TYPE", targetUpper)] = o.Workspace.Type.String() - envVars[fmt.Sprintf("DBT_KBC_%s_SCHEMA", targetUpper)] = o.Workspace.Details.Connection.Schema - envVars[fmt.Sprintf("DBT_KBC_%s_WAREHOUSE", targetUpper)] = o.Workspace.Details.Connection.Warehouse - envVars[fmt.Sprintf("DBT_KBC_%s_DATABASE", targetUpper)] = o.Workspace.Details.Connection.Database + envVars[fmt.Sprintf("DBT_KBC_%s_TYPE", targetUpper)] = o.Workspace.Type + envVars[fmt.Sprintf("DBT_KBC_%s_SCHEMA", targetUpper)] = o.Workspace.Schema + envVars[fmt.Sprintf("DBT_KBC_%s_WAREHOUSE", targetUpper)] = o.Workspace.Warehouse + envVars[fmt.Sprintf("DBT_KBC_%s_DATABASE", targetUpper)] = o.Workspace.Database - linkedBucketEnvsMap := make(map[string]string) // Store env var name -> value + linkedBucketEnvsMap := make(map[string]string) for _, bucket := range o.Buckets { if bucket.LinkedProjectID != 0 { envVarName := bucket.DatabaseEnv if _, exists := linkedBucketEnvsMap[envVarName]; !exists { - stackPrefix, _, _ := strings.Cut(o.Workspace.Details.Connection.Database, "_") // SAPI_..., KEBOOLA_..., etc. + stackPrefix, _, _ := strings.Cut(o.Workspace.Database, "_") envVarValue := fmt.Sprintf("%s_%d", stackPrefix, bucket.LinkedProjectID) linkedBucketEnvsMap[envVarName] = envVarValue envVars[envVarName] = envVarValue @@ -80,8 +90,6 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) { } envVars[fmt.Sprintf("DBT_KBC_%s_ACCOUNT", targetUpper)] = host envVars[fmt.Sprintf("DBT_KBC_%s_USER", targetUpper)] = o.Workspace.User - // Only add private key if key-pair authentication was explicitly requested - // This ensures password-only workspaces don't get a private key in .env.local if o.UseKeyPair && len(o.PrivateKey) > 0 { envVars[fmt.Sprintf("DBT_KBC_%s_PRIVATE_KEY", targetUpper)] = o.PrivateKey } @@ -89,20 +97,23 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) { envVars[fmt.Sprintf("DBT_KBC_%s_PASSWORD", targetUpper)] = o.Workspace.Password } - // Format KEY=VALUE pairs - // Sort keys for consistent order + // Keboola adapter vars — written when the workspace was created via an editor session. + if len(o.Workspace.BaseURL) > 0 { + envVars[fmt.Sprintf("DBT_KBC_%s_BASE_URL", targetUpper)] = o.Workspace.BaseURL + envVars[fmt.Sprintf("DBT_KBC_%s_BRANCH_ID", targetUpper)] = o.Workspace.BranchID.String() + envVars[fmt.Sprintf("DBT_KBC_%s_WORKSPACE_ID", targetUpper)] = o.Workspace.WorkspaceID + } + + // Sort keys for consistent output. keys := make([]string, 0, len(envVars)) for k := range envVars { keys = append(keys, k) } sort.Strings(keys) + var envContent strings.Builder for _, k := range keys { v := envVars[k] - // Normalize multiline/special values for dotenv compatibility: - // - Replace newlines and carriage returns by literal \n to keep a single line per var - // - Escape existing double quotes - // - Wrap in double quotes if any special characters present hasSpecial := strings.ContainsAny(v, " #\"'\\\n\r\t") if strings.Contains(v, "\n") || strings.Contains(v, "\r") { v = strings.ReplaceAll(v, "\r\n", "\n") @@ -116,14 +127,12 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) { _, _ = fmt.Fprintf(&envContent, "%s=%s\n", k, v) } - // Write to .env.local envFilePath := filesystem.Join(dbtProject.Fs().WorkingDir(), ".env.local") envFile := filesystem.NewRawFile(envFilePath, envContent.String()).SetDescription("dbt environment variables") if err := d.Fs().WriteFile(ctx, envFile); err != nil { return errors.Errorf("cannot write file \"%s\": %w", envFilePath, err) } - // Print info message l := d.Logger() l.Infof(ctx, `Environment variables for dbt target "%s" have been written to "%s".`, o.TargetName, envFilePath) l.Info(ctx, `To load the variables into your current shell session, run:`) diff --git a/pkg/lib/operation/dbt/generate/profile/operation.go b/pkg/lib/operation/dbt/generate/profile/operation.go index 1e89f84e49..6628ce9e50 100644 --- a/pkg/lib/operation/dbt/generate/profile/operation.go +++ b/pkg/lib/operation/dbt/generate/profile/operation.go @@ -54,6 +54,7 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) { // Set profile targetUpper := strings.ToUpper(o.TargetName) + keboolaTargetName := "keboola_" + o.TargetName profilesFile.Set(project.Profile(), orderedmap.FromPairs([]orderedmap.Pair{ { Key: "target", @@ -106,6 +107,45 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) { }, }), }, + // keboola_snowflake adapter target — uses Keboola Query Service instead of direct Snowflake access. + // Run with: dbt run --target keboola_{target_name} + { + Key: keboolaTargetName, + Value: orderedmap.FromPairs([]orderedmap.Pair{ + { + Key: "type", + Value: "keboola_snowflake", + }, + { + Key: "base_url", + Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_BASE_URL\") }}", targetUpper), + }, + { + Key: "token", + Value: "{{ env_var(\"KEBOOLA_TOKEN\") }}", + }, + { + Key: "branch_id", + Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_BRANCH_ID\") }}", targetUpper), + }, + { + Key: "workspace_id", + Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_WORKSPACE_ID\") }}", targetUpper), + }, + { + Key: "database", + Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_DATABASE\") }}", targetUpper), + }, + { + Key: "schema", + Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_SCHEMA\") }}", targetUpper), + }, + { + Key: "warehouse", + Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_WAREHOUSE\") }}", targetUpper), + }, + }), + }, }), }, })) diff --git a/pkg/lib/operation/dbt/init/operation.go b/pkg/lib/operation/dbt/init/operation.go index 4469d6b1c5..b23c12449c 100644 --- a/pkg/lib/operation/dbt/init/operation.go +++ b/pkg/lib/operation/dbt/init/operation.go @@ -9,7 +9,6 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/dbt" "github.com/keboola/keboola-as-code/internal/pkg/filesystem" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" @@ -24,6 +23,7 @@ type DbtInitOptions struct { TargetName string WorkspaceName string UseKeyPair bool + BaseURL string // Keboola Query Service base URL, e.g. "https://query.keboola.com" } type dependencies interface { @@ -69,8 +69,24 @@ func Run(ctx context.Context, o DbtInitOptions, d dependencies) (err error) { return errors.Errorf("cannot fetch workspace credentials: %w", err) } - // Build SandboxWorkspace from StorageWorkspace for env generation. - workspace := sandbox.WorkspaceFromStorage(storageWS, keboola.SandboxWorkspaceType(storageWS.StorageWorkspaceDetails.Backend)) + // Build WorkspaceDetails from StorageWorkspace credentials. + deref := func(s *string) string { + if s == nil { + return "" + } + return *s + } + workspace := env.WorkspaceDetails{ + Type: string(storageWS.StorageWorkspaceDetails.Backend), + Host: deref(storageWS.StorageWorkspaceDetails.Host), + User: deref(storageWS.StorageWorkspaceDetails.User), + Database: deref(storageWS.StorageWorkspaceDetails.Database), + Schema: deref(storageWS.StorageWorkspaceDetails.Schema), + Warehouse: deref(storageWS.StorageWorkspaceDetails.Warehouse), + BaseURL: o.BaseURL, + BranchID: branch.ID, + WorkspaceID: session.EditorSession.WorkspaceID, + } // Determine private key from the freshly created credentials. privateKey := "" diff --git a/pkg/lib/operation/project/remote/workspace/create/operation.go b/pkg/lib/operation/project/remote/workspace/create/operation.go index 8635145a4a..8c5e9bd12f 100644 --- a/pkg/lib/operation/project/remote/workspace/create/operation.go +++ b/pkg/lib/operation/project/remote/workspace/create/operation.go @@ -6,15 +6,15 @@ import ( "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + workspace "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" ) type CreateOptions struct { Name string - Type keboola.SandboxWorkspaceType + Type workspace.WorkspaceType Size string } @@ -40,17 +40,13 @@ func Run(ctx context.Context, o CreateOptions, d dependencies) (err error) { logger.Info(ctx, `Creating a new workspace, please wait.`) - if keboola.SandboxWorkspaceSupportsSizes(o.Type) { - // Python/R workspace - opts := make([]sandbox.CreateSandboxWorkspaceOption, 0) - if len(o.Size) > 0 { - opts = append(opts, sandbox.WithSize(o.Size)) - } - w, err := sandbox.CreateSandboxWorkspace(ctx, d.KeboolaProjectAPI(), branch.ID, o.Name, o.Type, opts...) + if workspace.WorkspaceSupportsSizes(o.Type) { + // Python/R workspace: create config then call DataScience sandbox endpoint. + configID, err := createPyRWorkspace(ctx, d.KeboolaProjectAPI(), branch.ID, o.Name, o.Type, o.Size) if err != nil { return errors.Errorf("cannot create workspace: %w", err) } - logger.Infof(ctx, `Created the new workspace "%s" (%s).`, o.Name, w.Config.ID) + logger.Infof(ctx, `Created the new workspace "%s" (%s).`, o.Name, configID) } else { // SQL workspace (Snowflake/BigQuery) — backend determined by project config session, err := d.KeboolaProjectAPI().CreateEditorSession(ctx, branch.ID, o.Name) @@ -62,3 +58,25 @@ func Run(ctx context.Context, o CreateOptions, d dependencies) (err error) { return nil } + +// createPyRWorkspace creates a Python/R workspace: creates the sandboxes config, then calls +// the DataScience sandbox service to provision the instance. +func createPyRWorkspace(ctx context.Context, api *keboola.AuthorizedAPI, branchID keboola.BranchID, name string, wsType workspace.WorkspaceType, size string) (keboola.ConfigID, error) { + config, err := api.CreateSandboxWorkspaceConfigRequest(branchID, name).Send(ctx) + if err != nil { + return "", err + } + + _, err = api.CreateDataScienceSandboxRequest(keboola.CreateDataScienceSandboxPayload{ + Type: keboola.DataScienceAppType(wsType), + ConfigurationID: string(config.ID), + ComponentID: string(keboola.SandboxWorkspacesComponent), + BranchID: branchID.String(), + Size: keboola.DataScienceSandboxSize(size), + }).Send(ctx) + if err != nil { + return "", err + } + + return config.ID, nil +} diff --git a/pkg/lib/operation/project/remote/workspace/delete/operation.go b/pkg/lib/operation/project/remote/workspace/delete/operation.go index d1f7c76991..c02fb25537 100644 --- a/pkg/lib/operation/project/remote/workspace/delete/operation.go +++ b/pkg/lib/operation/project/remote/workspace/delete/operation.go @@ -6,10 +6,10 @@ import ( "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + wsinfo "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" ) type dependencies interface { @@ -18,10 +18,9 @@ type dependencies interface { KeboolaProjectAPI() *keboola.AuthorizedAPI } -// Run deletes a workspace. For Python/R workspaces, it uses the sandbox delete API. -// For SQL (Snowflake/BigQuery) workspaces created via editor sessions, workspace.SandboxWorkspace.ID -// holds the EditorSessionID and deletion goes through the editor session API. -func Run(ctx context.Context, d dependencies, branchID keboola.BranchID, workspace *sandbox.SandboxWorkspaceWithConfig) (err error) { +// Run deletes a workspace. For Python/R workspaces it runs a delete queue job then removes the config. +// For SQL workspaces it calls DeleteEditorSession. +func Run(ctx context.Context, d dependencies, branchID keboola.BranchID, workspace *wsinfo.WorkspaceWithConfig) (err error) { ctx, span := d.Telemetry().Tracer().Start(ctx, "keboola.go.operation.project.remote.workspace.delete") defer span.End(&err) @@ -32,12 +31,12 @@ func Run(ctx context.Context, d dependencies, branchID keboola.BranchID, workspa logger.Infof(ctx, `Deleting the workspace "%s" (%s), please wait.`, workspace.Config.Name, workspace.Config.ID) - if keboola.SandboxWorkspaceSupportsSizes(workspace.SandboxWorkspace.Type) { - // Python/R workspace - err = sandbox.DeleteSandboxWorkspace(ctx, d.KeboolaProjectAPI(), branchID, workspace.Config.ID, workspace.SandboxWorkspace.ID) + if workspace.App != nil { + // Python/R workspace: delete via DataScience sandbox endpoint, then delete the config. + err = deletePyRWorkspace(ctx, d.KeboolaProjectAPI(), branchID, workspace.Config.ID, workspace.App.ID) } else { - // SQL workspace (Snowflake/BigQuery) — SandboxWorkspace.ID stores the EditorSessionID - err = d.KeboolaProjectAPI().DeleteEditorSession(ctx, branchID, workspace.Config.ID, keboola.EditorSessionID(workspace.SandboxWorkspace.ID)) + // SQL workspace (Snowflake/BigQuery) — Session.ID is the EditorSessionID. + err = d.KeboolaProjectAPI().DeleteEditorSession(ctx, branchID, workspace.Config.ID, workspace.Session.ID) } if err != nil { return err @@ -46,3 +45,12 @@ func Run(ctx context.Context, d dependencies, branchID keboola.BranchID, workspa logger.Infof(ctx, "Delete done.") return nil } + +// deletePyRWorkspace deletes the DataScience sandbox instance then removes the config. +func deletePyRWorkspace(ctx context.Context, api *keboola.AuthorizedAPI, branchID keboola.BranchID, configID keboola.ConfigID, appID keboola.DataScienceAppID) error { + if _, err := api.DeleteDataScienceSandboxRequest(appID).Send(ctx); err != nil { + return err + } + _, err := api.DeleteSandboxWorkspaceConfigRequest(branchID, configID).Send(ctx) + return err +} diff --git a/pkg/lib/operation/project/remote/workspace/detail/operation.go b/pkg/lib/operation/project/remote/workspace/detail/operation.go index ec6e6c8728..41531fb0b8 100644 --- a/pkg/lib/operation/project/remote/workspace/detail/operation.go +++ b/pkg/lib/operation/project/remote/workspace/detail/operation.go @@ -6,10 +6,10 @@ import ( "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + wsinfo "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" ) type dependencies interface { @@ -39,8 +39,10 @@ func Run(ctx context.Context, d dependencies, configID keboola.ConfigID) (err er } // Check if this is a Python/R workspace (has parameters.id) or a SQL editor session. - _, wsIDErr := sandbox.GetSandboxWorkspaceID(config) - if wsIDErr != nil { + // GetNested returns an error when an intermediate path key is missing, so treat + // both err != nil and !found the same way: fall through to editor-session lookup. + wsID, found, err := config.Content.GetNested("parameters.id") + if err != nil || !found { // SQL workspace — find the editor session linked to this config. sessions, e := d.KeboolaProjectAPI().ListEditorSessionsRequest().Send(ctx) if e != nil { @@ -56,21 +58,23 @@ func Run(ctx context.Context, d dependencies, configID keboola.ConfigID) (err er return errors.Errorf(`no active editor session found for workspace "%s"`, configID) } - // Python/R workspace - workspace, err := sandbox.GetSandboxWorkspace(ctx, d.KeboolaProjectAPI(), branch.ID, configID) + // Python/R workspace — fetch DataScienceApp for credentials. + workspaceIDStr, ok := wsID.(string) + if !ok { + return errors.Errorf("config.parameters.id is not a string") + } + + app, err := d.KeboolaProjectAPI().GetDataScienceAppRequest(keboola.DataScienceAppID(workspaceIDStr)).Send(ctx) if err != nil { return err } - c, w := workspace.Config, workspace.SandboxWorkspace - - logger.Infof(ctx, "Workspace \"%s\"\nID: %s\nType: %s", c.Name, c.ID, w.Type) - if keboola.SandboxWorkspaceSupportsSizes(w.Type) { - logger.Infof(ctx, `Size: %s`, w.Size) + logger.Infof(ctx, "Workspace \"%s\"\nID: %s\nType: %s", config.Name, config.ID, app.Type) + if wsinfo.WorkspaceSupportsSizes(string(app.Type)) { + logger.Infof(ctx, `Size: %s`, app.Size) } - - if w.Host != "" || w.Password != "" { - logger.Infof(ctx, "Credentials:\n Host: %s\n Password: %s", w.Host, w.Password) + if app.URL != "" { + logger.Infof(ctx, "URL: %s", app.URL) } return nil diff --git a/pkg/lib/operation/project/remote/workspace/list/operation.go b/pkg/lib/operation/project/remote/workspace/list/operation.go index 5a14f4531e..456082c091 100644 --- a/pkg/lib/operation/project/remote/workspace/list/operation.go +++ b/pkg/lib/operation/project/remote/workspace/list/operation.go @@ -5,11 +5,12 @@ import ( "sort" "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" + "golang.org/x/sync/errgroup" - "github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox" "github.com/keboola/keboola-as-code/internal/pkg/log" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + wsinfo "github.com/keboola/keboola-as-code/pkg/lib/operation/project/remote/workspace" ) type dependencies interface { @@ -31,21 +32,110 @@ func Run(ctx context.Context, d dependencies) (err error) { logger.Info(ctx, "Loading workspaces, please wait.") - all, _, err := sandbox.ListAllWorkspaces(ctx, d.KeboolaProjectAPI(), branch.ID) - if err != nil { + // Fetch Python/R workspaces (via DataScienceApp) and SQL editor sessions in parallel. + var pyRWorkspaces []*wsinfo.WorkspaceWithConfig + var allConfigs []*keboola.Config + var sessions []*keboola.EditorSession + + grp, grpCtx := errgroup.WithContext(ctx) + grp.Go(func() error { + var e error + pyRWorkspaces, allConfigs, e = ListPyRWorkspaces(grpCtx, d.KeboolaProjectAPI(), branch.ID) + return e + }) + grp.Go(func() error { + result, e := d.KeboolaProjectAPI().ListEditorSessionsRequest().Send(grpCtx) + if e != nil { + return e + } + sessions = *result + return nil + }) + if err := grp.Wait(); err != nil { return err } + // Build config name map for editor session name lookup. + configNameMap := make(map[string]string) + for _, c := range allConfigs { + configNameMap[c.ID.String()] = c.Name + } + + // Build combined list: Python/R workspaces + SQL editor sessions. + all := make([]*wsinfo.WorkspaceWithConfig, 0, len(pyRWorkspaces)+len(sessions)) + all = append(all, pyRWorkspaces...) + for _, s := range sessions { + name := configNameMap[s.ConfigurationID] + all = append(all, &wsinfo.WorkspaceWithConfig{ + Config: &keboola.Config{ + ConfigKey: keboola.ConfigKey{ID: keboola.ConfigID(s.ConfigurationID)}, + Name: name, + }, + Session: s, + }) + } + + sort.Slice(all, func(i, j int) bool { return all[i].Config.Name < all[j].Config.Name }) logger.Info(ctx, "Found workspaces:") for _, workspace := range all { - if keboola.SandboxWorkspaceSupportsSizes(workspace.SandboxWorkspace.Type) { - logger.Infof(ctx, " %s (ID: %s, Type: %s, Size: %s)", workspace.Config.Name, workspace.Config.ID, workspace.SandboxWorkspace.Type, workspace.SandboxWorkspace.Size) + if workspace.SupportsSizes() { + logger.Infof(ctx, " %s (ID: %s, Type: %s, Size: %s)", workspace.Config.Name, workspace.Config.ID, workspace.Type(), workspace.Size()) } else { - logger.Infof(ctx, " %s (ID: %s, Type: %s)", workspace.Config.Name, workspace.Config.ID, workspace.SandboxWorkspace.Type) + logger.Infof(ctx, " %s (ID: %s, Type: %s)", workspace.Config.Name, workspace.Config.ID, workspace.Type()) } } return nil } + +// ListPyRWorkspaces fetches Python/R workspaces by listing DataScienceApps and their configs in parallel, +// joining by ConfigID. It also returns all sandbox configs so callers can look up SQL workspace names. +func ListPyRWorkspaces(ctx context.Context, api *keboola.AuthorizedAPI, branchID keboola.BranchID) ([]*wsinfo.WorkspaceWithConfig, []*keboola.Config, error) { + var configs []*keboola.Config + var apps []*keboola.DataScienceApp + + grp, grpCtx := errgroup.WithContext(ctx) + grp.Go(func() error { + data, e := api.ListSandboxWorkspaceConfigRequest(branchID).Send(grpCtx) + if e != nil { + return e + } + configs = *data + return nil + }) + grp.Go(func() error { + data, e := api.ListDataScienceAppsRequest( + keboola.WithDataScienceAppsComponentID(keboola.ComponentID(keboola.SandboxWorkspacesComponent)), + keboola.WithDataScienceAppsBranchID(branchID.String()), + keboola.WithDataScienceAppsType(keboola.DataScienceAppTypePython, keboola.DataScienceAppTypeR), + ).Send(grpCtx) + if e != nil { + return e + } + apps = *data + return nil + }) + if err := grp.Wait(); err != nil { + return nil, nil, err + } + + appsByConfigID := make(map[string]*keboola.DataScienceApp, len(apps)) + for _, app := range apps { + appsByConfigID[app.ConfigID] = app + } + + out := make([]*wsinfo.WorkspaceWithConfig, 0) + for _, config := range configs { + app, found := appsByConfigID[config.ID.String()] + if !found { + continue + } + out = append(out, &wsinfo.WorkspaceWithConfig{ + App: app, + Config: config, + }) + } + return out, configs, nil +} diff --git a/pkg/lib/operation/project/remote/workspace/workspaceinfo.go b/pkg/lib/operation/project/remote/workspace/workspaceinfo.go new file mode 100644 index 0000000000..6009583b51 --- /dev/null +++ b/pkg/lib/operation/project/remote/workspace/workspaceinfo.go @@ -0,0 +1,95 @@ +package workspace + +import ( + "fmt" + + "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" +) + +// WorkspaceType is a string alias for the workspace type (snowflake, bigquery, python, r). +// Defined locally because the SDK no longer provides a combined type for all workspace backends. +type WorkspaceType = string + +const ( + WorkspaceTypeSnowflake WorkspaceType = "snowflake" + WorkspaceTypeBigQuery WorkspaceType = "bigquery" + WorkspaceTypePython WorkspaceType = "python" + WorkspaceTypeR WorkspaceType = "r" +) + +// WorkspaceSupportsSizes reports whether the given workspace type supports size selection. +func WorkspaceSupportsSizes(typ WorkspaceType) bool { + return keboola.DataScienceSandboxSupportsSizes(keboola.DataScienceAppType(typ)) +} + +// WorkspaceTypesOrdered returns all workspace types in a stable display order. +func WorkspaceTypesOrdered() []WorkspaceType { + return []WorkspaceType{WorkspaceTypeSnowflake, WorkspaceTypeBigQuery, WorkspaceTypePython, WorkspaceTypeR} +} + +// WorkspaceTypesMap returns a set of all valid workspace types. +func WorkspaceTypesMap() map[WorkspaceType]bool { + m := make(map[WorkspaceType]bool, len(WorkspaceTypesOrdered())) + for _, t := range WorkspaceTypesOrdered() { + m[t] = true + } + return m +} + +// WorkspaceSizesOrdered returns sandbox sizes in ascending order. +func WorkspaceSizesOrdered() []string { + sizes := keboola.DataScienceSandboxSizesOrdered() + result := make([]string, len(sizes)) + for i, s := range sizes { + result[i] = string(s) + } + return result +} + +// WorkspaceSizesMap returns the set of valid sandbox sizes. +func WorkspaceSizesMap() map[string]bool { + m := make(map[string]bool) + for _, s := range keboola.DataScienceSandboxSizesOrdered() { + m[string(s)] = true + } + return m +} + +// WorkspaceWithConfig pairs a workspace instance with its keboola.sandboxes component config. +// For Python/R workspaces, App is set and Session is nil. +// For SQL (Snowflake/BigQuery) workspaces created via editor sessions, Session is set and App is nil. +type WorkspaceWithConfig struct { + App *keboola.DataScienceApp // non-nil for Python/R + Session *keboola.EditorSession // non-nil for SQL + Config *keboola.Config // always set +} + +func (w *WorkspaceWithConfig) Type() WorkspaceType { + if w.App != nil { + return string(w.App.Type) + } + if w.Session != nil { + return string(w.Session.BackendType) + } + return "" +} + +func (w *WorkspaceWithConfig) Size() string { + if w.App != nil { + return w.App.Size + } + return "" +} + +func (w *WorkspaceWithConfig) SupportsSizes() bool { + return WorkspaceSupportsSizes(w.Type()) +} + +func (w *WorkspaceWithConfig) String() string { + if w.SupportsSizes() { + return fmt.Sprintf("ID: %s, Type: %s, Size: %s, Name: %s", + w.Config.ID, w.Type(), w.Size(), w.Config.Name) + } + return fmt.Sprintf("ID: %s, Type: %s, Name: %s", + w.Config.ID, w.Type(), w.Config.Name) +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/expected-stderr b/test/cli/allow-target-env/pull-ignore-configs/expected-stderr index e69de29bb2..56002fd67b 100644 --- a/test/cli/allow-target-env/pull-ignore-configs/expected-stderr +++ b/test/cli/allow-target-env/pull-ignore-configs/expected-stderr @@ -0,0 +1,8 @@ +Warning: +- Validation failed: + - Config "main/extractor/keboola.ex-db-mysql/with-rows/config.json" doesn't match schema: + - "db": missing properties "port", "user", "#password" + +The project has been pulled, but it is not in a valid state. +Please correct the problems listed above. +Push will proceed with warnings for configs already accepted by the server. diff --git a/test/cli/push/delete-config-row-dev-force/expected-stderr b/test/cli/push/delete-config-row-dev-force/expected-stderr index e69de29bb2..127f6b144d 100644 --- a/test/cli/push/delete-config-row-dev-force/expected-stderr +++ b/test/cli/push/delete-config-row-dev-force/expected-stderr @@ -0,0 +1,7 @@ +Warning: +- Validation failed: + - Config "foo/extractor/keboola.ex-db-mysql/with-rows/config.json" doesn't match schema: + - "db": missing properties "port", "user", "#password" + +Validation warnings found, proceeding with push. +The configs already exist on the server; the server is the source of truth. diff --git a/test/cli/template-test-create/create-empty-tests-folder/expected-stderr b/test/cli/template-test-create/create-empty-tests-folder/expected-stderr index e69de29bb2..b48254caea 100644 --- a/test/cli/template-test-create/create-empty-tests-folder/expected-stderr +++ b/test/cli/template-test-create/create-empty-tests-folder/expected-stderr @@ -0,0 +1,7 @@ +Warning: +- Validation failed: + - Config "main/extractor/keboola.ex-db-mysql/with-rows/config.json" doesn't match schema: + - "db": missing properties "port", "user" + +Please correct the problems listed above. +Push operation is only possible when project is valid. diff --git a/test/cli/template-test-create/create-folder/expected-stderr b/test/cli/template-test-create/create-folder/expected-stderr index e69de29bb2..b48254caea 100644 --- a/test/cli/template-test-create/create-folder/expected-stderr +++ b/test/cli/template-test-create/create-folder/expected-stderr @@ -0,0 +1,7 @@ +Warning: +- Validation failed: + - Config "main/extractor/keboola.ex-db-mysql/with-rows/config.json" doesn't match schema: + - "db": missing properties "port", "user" + +Please correct the problems listed above. +Push operation is only possible when project is valid. diff --git a/test/cli/template-test-create/create-test-two/expected-stderr b/test/cli/template-test-create/create-test-two/expected-stderr index e69de29bb2..b48254caea 100644 --- a/test/cli/template-test-create/create-test-two/expected-stderr +++ b/test/cli/template-test-create/create-test-two/expected-stderr @@ -0,0 +1,7 @@ +Warning: +- Validation failed: + - Config "main/extractor/keboola.ex-db-mysql/with-rows/config.json" doesn't match schema: + - "db": missing properties "port", "user" + +Please correct the problems listed above. +Push operation is only possible when project is valid.