Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions internal/pkg/service/cli/cmd/dbt/dbtinit/cmd.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package dbtinit

import (
"strings"

"github.com/spf13/cobra"

"github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies"
Expand Down Expand Up @@ -57,6 +59,7 @@ func Command(p dependencies.Provider) *cobra.Command {
if err != nil {
return err
}
opts.BaseURL = baseURLFromHost(d.StorageAPIHost())

// Send cmd successful/failed event
defer d.EventSender().SendCmdEvent(cmd.Context(), d.Clock().Now(), &cmdErr, "dbt-init")
Expand All @@ -69,3 +72,10 @@ func Command(p dependencies.Provider) *cobra.Command {

return cmd
}

// baseURLFromHost derives the Keboola Query Service URL from the Storage API host.
// "https://connection.keboola.com" → "https://query.keboola.com"
func baseURLFromHost(host string) string {
bare := strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://")
return "https://query." + strings.TrimPrefix(bare, "connection.")
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
}
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
12 changes: 12 additions & 0 deletions internal/pkg/service/cli/cmd/dbt/generate/env/cmd.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package env

import (
"strings"

"github.com/spf13/cobra"

"github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox"
Expand Down Expand Up @@ -61,6 +63,9 @@ func Command(p dependencies.Provider) *cobra.Command {
return err
}

// Set BaseURL for keboola adapter vars (only written when WorkspaceID is also set).
opts.Workspace.BaseURL = baseURLFromHost(d.StorageAPIHost())

// Send cmd successful/failed event
defer d.EventSender().SendCmdEvent(cmd.Context(), d.Clock().Now(), &cmdErr, "dbt-generate-env")

Expand All @@ -72,3 +77,10 @@ func Command(p dependencies.Provider) *cobra.Command {

return cmd
}

// baseURLFromHost derives the Keboola Query Service URL from the Storage API host.
// "https://connection.keboola.com" → "https://query.keboola.com"
func baseURLFromHost(host string) string {
bare := strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://")
return "https://query." + strings.TrimPrefix(bare, "connection.")
}
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
36 changes: 28 additions & 8 deletions internal/pkg/service/cli/cmd/dbt/generate/env/dialog.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,25 @@ func AskGenerateEnv(
useKeyPair := len(privateKey) > 0

if keboola.SandboxWorkspaceSupportsSizes(workspace.SandboxWorkspace.Type) {
// Python/R workspace — use credentials directly.
// Python/R workspace — credential fields are not available from the listing.
return genenv.Options{
BranchKey: branchKey,
TargetName: targetName,
Workspace: workspace.SandboxWorkspace, // already *sandbox.SandboxWorkspace
Workspace: genenv.WorkspaceDetails{
Type: string(workspace.SandboxWorkspace.Type),
},
UseKeyPair: useKeyPair,
PrivateKey: privateKey,
}, nil
Comment thread
Matovidlo marked this conversation as resolved.
}

// SQL workspace (Snowflake/BigQuery) — fetch StorageWorkspace credentials via the editor session.
// Find the editor session for this workspace by matching ConfigurationID.
// Phase 1 (keboola_snowflake profile): editor session coordinates are already available
// in the matched session (WorkspaceID, BranchID set below). No credential rotation needed.

// Phase 2 (direct-Snowflake profile): fetch storage workspace credentials — server
// generates a keypair, registers the public key with the workspace, and returns the
// private key together with all connection details (Host, User, DB, Schema, Warehouse).
// Password auth is deprecated; keypair is used instead.
var matchedSession *keboola.EditorSession
for _, s := range sessions {
if s.ConfigurationID == workspace.Config.ID.String() {
Expand All @@ -74,14 +81,27 @@ func AskGenerateEnv(
return genenv.Options{}, errors.Errorf("cannot parse workspace ID %q: %w", matchedSession.WorkspaceID, err)
}

// StorageWorkspaceCreateCredentialsRequest creates new credentials on each call,
// which rotates any previously issued credentials for this workspace.
storageWS, err := api.StorageWorkspaceCreateCredentialsRequest(branchID, workspaceIDUint).Send(ctx)
if err != nil {
return genenv.Options{}, errors.Errorf("cannot fetch workspace credentials: %w", err)
}

sandboxWS := sandbox.WorkspaceFromStorage(storageWS, keboola.SandboxWorkspaceType(matchedSession.BackendType))
deref := func(s *string) string {
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
if s == nil {
return ""
}
return *s
}
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
ws := genenv.WorkspaceDetails{
Type: string(matchedSession.BackendType),
Host: deref(storageWS.StorageWorkspaceDetails.Host),
User: deref(storageWS.StorageWorkspaceDetails.User),
Database: deref(storageWS.StorageWorkspaceDetails.Database),
Schema: deref(storageWS.StorageWorkspaceDetails.Schema),
Warehouse: deref(storageWS.StorageWorkspaceDetails.Warehouse),
BranchID: branchID,
WorkspaceID: matchedSession.WorkspaceID,
}
Comment thread
Matovidlo marked this conversation as resolved.

// Use server-provided private key for SQL workspaces when available.
if len(privateKey) == 0 && storageWS.StorageWorkspaceDetails.PrivateKey != nil && len(*storageWS.StorageWorkspaceDetails.PrivateKey) > 0 {
Expand All @@ -92,7 +112,7 @@ func AskGenerateEnv(
return genenv.Options{
BranchKey: branchKey,
TargetName: targetName,
Workspace: sandboxWS,
Workspace: ws,
UseKeyPair: useKeyPair,
PrivateKey: privateKey,
}, nil
Expand Down
48 changes: 33 additions & 15 deletions pkg/lib/operation/dbt/generate/env/operation.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,32 @@ import (

"github.com/keboola/keboola-as-code/internal/pkg/dbt"
"github.com/keboola/keboola-as-code/internal/pkg/filesystem"
"github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox"
"github.com/keboola/keboola-as-code/internal/pkg/log"
"github.com/keboola/keboola-as-code/internal/pkg/telemetry"
"github.com/keboola/keboola-as-code/internal/pkg/utils/errors"
"github.com/keboola/keboola-as-code/pkg/lib/operation/dbt/listbuckets"
)

// WorkspaceDetails holds the connection details for a workspace used in dbt env generation.
type WorkspaceDetails struct {
Type string // workspace type string, e.g. "snowflake", "python"
Host string
User string
Password string //nolint:gosec
Database string
Schema string
Warehouse string
// Fields for the keboola_snowflake dbt adapter (populated for SQL workspaces).
// When both BaseURL and WorkspaceID are set, DBT_KBC_{TARGET}_BASE_URL / _BRANCH_ID / _WORKSPACE_ID are written.
BaseURL string // e.g. "https://query.keboola.com"
BranchID keboola.BranchID
WorkspaceID string // numeric workspace ID from EditorSession
}

type Options struct {
BranchKey keboola.BranchKey
TargetName string
Workspace *sandbox.SandboxWorkspace
Workspace WorkspaceDetails
PrivateKey string //nolint:gosec
UseKeyPair bool // Whether key-pair authentication was requested (only add private key if true)
Buckets []listbuckets.Bucket // optional, set if the buckets have been loaded in a parent command
Expand Down Expand Up @@ -53,25 +68,23 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) {

targetUpper := strings.ToUpper(o.TargetName)
host := o.Workspace.Host
if o.Workspace.Type == keboola.SandboxWorkspaceTypeSnowflake {
if o.Workspace.Type == "snowflake" {
host = strings.Replace(host, ".snowflakecomputing.com", "", 1)
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
}

// Prepare content for .env.local
var envContent strings.Builder
envVars := make(map[string]string)

envVars[fmt.Sprintf("DBT_KBC_%s_TYPE", targetUpper)] = o.Workspace.Type.String()
envVars[fmt.Sprintf("DBT_KBC_%s_SCHEMA", targetUpper)] = o.Workspace.Details.Connection.Schema
envVars[fmt.Sprintf("DBT_KBC_%s_WAREHOUSE", targetUpper)] = o.Workspace.Details.Connection.Warehouse
envVars[fmt.Sprintf("DBT_KBC_%s_DATABASE", targetUpper)] = o.Workspace.Details.Connection.Database
envVars[fmt.Sprintf("DBT_KBC_%s_TYPE", targetUpper)] = o.Workspace.Type
envVars[fmt.Sprintf("DBT_KBC_%s_SCHEMA", targetUpper)] = o.Workspace.Schema
envVars[fmt.Sprintf("DBT_KBC_%s_WAREHOUSE", targetUpper)] = o.Workspace.Warehouse
envVars[fmt.Sprintf("DBT_KBC_%s_DATABASE", targetUpper)] = o.Workspace.Database

linkedBucketEnvsMap := make(map[string]string) // Store env var name -> value
linkedBucketEnvsMap := make(map[string]string)
for _, bucket := range o.Buckets {
if bucket.LinkedProjectID != 0 {
envVarName := bucket.DatabaseEnv
if _, exists := linkedBucketEnvsMap[envVarName]; !exists {
stackPrefix, _, _ := strings.Cut(o.Workspace.Details.Connection.Database, "_") // SAPI_..., KEBOOLA_..., etc.
stackPrefix, _, _ := strings.Cut(o.Workspace.Database, "_") // SAPI_..., KEBOOLA_..., etc.
envVarValue := fmt.Sprintf("%s_%d", stackPrefix, bucket.LinkedProjectID)
linkedBucketEnvsMap[envVarName] = envVarValue
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
envVars[envVarName] = envVarValue
Expand All @@ -80,23 +93,28 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) {
}
envVars[fmt.Sprintf("DBT_KBC_%s_ACCOUNT", targetUpper)] = host
envVars[fmt.Sprintf("DBT_KBC_%s_USER", targetUpper)] = o.Workspace.User
// Only add private key if key-pair authentication was explicitly requested
// This ensures password-only workspaces don't get a private key in .env.local
if o.UseKeyPair && len(o.PrivateKey) > 0 {
envVars[fmt.Sprintf("DBT_KBC_%s_PRIVATE_KEY", targetUpper)] = o.PrivateKey
}
if len(o.Workspace.Password) > 0 {
envVars[fmt.Sprintf("DBT_KBC_%s_PASSWORD", targetUpper)] = o.Workspace.Password
}

// Format KEY=VALUE pairs
// Sort keys for consistent order
// Keboola adapter vars — written when the workspace was created via an editor session.
if len(o.Workspace.BaseURL) > 0 && len(o.Workspace.WorkspaceID) > 0 {
envVars[fmt.Sprintf("DBT_KBC_%s_BASE_URL", targetUpper)] = o.Workspace.BaseURL
envVars[fmt.Sprintf("DBT_KBC_%s_BRANCH_ID", targetUpper)] = o.Workspace.BranchID.String()
envVars[fmt.Sprintf("DBT_KBC_%s_WORKSPACE_ID", targetUpper)] = o.Workspace.WorkspaceID
}
Comment thread
Matovidlo marked this conversation as resolved.
Comment thread
Matovidlo marked this conversation as resolved.

// Sort keys for consistent output.
keys := make([]string, 0, len(envVars))
for k := range envVars {
keys = append(keys, k)
}
sort.Strings(keys)

var envContent strings.Builder
for _, k := range keys {
v := envVars[k]
// Normalize multiline/special values for dotenv compatibility:
Expand Down
40 changes: 40 additions & 0 deletions pkg/lib/operation/dbt/generate/profile/operation.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) {

// Set profile
targetUpper := strings.ToUpper(o.TargetName)
keboolaTargetName := "keboola_" + o.TargetName
profilesFile.Set(project.Profile(), orderedmap.FromPairs([]orderedmap.Pair{
{
Key: "target",
Expand Down Expand Up @@ -106,6 +107,45 @@ func Run(ctx context.Context, o Options, d dependencies) (err error) {
},
}),
},
// keboola_snowflake adapter target — uses Keboola Query Service instead of direct Snowflake access.
// Run with: dbt run --target keboola_{target_name}
{
Key: keboolaTargetName,
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
Value: orderedmap.FromPairs([]orderedmap.Pair{
{
Key: "type",
Value: "keboola_snowflake",
},
{
Key: "base_url",
Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_BASE_URL\") }}", targetUpper),
},
{
Key: "token",
Value: "{{ env_var(\"KEBOOLA_TOKEN\") }}",
},
{
Key: "branch_id",
Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_BRANCH_ID\") }}", targetUpper),
},
{
Key: "workspace_id",
Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_WORKSPACE_ID\") }}", targetUpper),
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
},
{
Key: "database",
Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_DATABASE\") }}", targetUpper),
},
{
Key: "schema",
Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_SCHEMA\") }}", targetUpper),
},
{
Key: "warehouse",
Value: fmt.Sprintf("{{ env_var(\"DBT_KBC_%s_WAREHOUSE\") }}", targetUpper),
},
}),
},
Comment thread
Matovidlo marked this conversation as resolved.
}),
},
}))
Expand Down
32 changes: 27 additions & 5 deletions pkg/lib/operation/dbt/init/operation.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (

"github.com/keboola/keboola-as-code/internal/pkg/dbt"
"github.com/keboola/keboola-as-code/internal/pkg/filesystem"
"github.com/keboola/keboola-as-code/internal/pkg/keboola/sandbox"
"github.com/keboola/keboola-as-code/internal/pkg/log"
"github.com/keboola/keboola-as-code/internal/pkg/telemetry"
"github.com/keboola/keboola-as-code/internal/pkg/utils/errors"
Expand All @@ -24,6 +23,7 @@ type DbtInitOptions struct {
TargetName string
WorkspaceName string
UseKeyPair bool
BaseURL string // Keboola Query Service base URL, e.g. "https://query.keboola.com"
}

type dependencies interface {
Expand Down Expand Up @@ -51,15 +51,19 @@ func Run(ctx context.Context, o DbtInitOptions, d dependencies) (err error) {
ctx, cancel := context.WithTimeoutCause(ctx, 10*time.Minute, errors.New("dbt init timeout"))
defer cancel()

// Create SQL workspace via editor session — backend is determined by the project config.
// Phase 1: Create editor session — provides workspace coordinates for the keboola_snowflake
// dbt profile (WorkspaceID, BranchID, BaseURL). No credential rotation at this step.
d.Logger().Info(ctx, `Creating a new workspace, please wait.`)
session, err := d.KeboolaProjectAPI().CreateEditorSession(ctx, branch.ID, o.WorkspaceName)
if err != nil {
return errors.Errorf("cannot create workspace: %w", err)
}
d.Logger().Infof(ctx, `Created the new workspace "%s".`, o.WorkspaceName)

// Create fresh credentials for the storage workspace to get connection details + private key.
// Phase 2: Create storage workspace credentials — server generates a keypair, registers
// the public key with the workspace, and returns the private key together with all
// connection details (Host, User, Database, Schema, Warehouse). These fill the
// direct-Snowflake dbt profile. Password auth is deprecated; keypair is used instead.
workspaceIDUint, err := strconv.ParseUint(session.EditorSession.WorkspaceID, 10, 64)
if err != nil {
return errors.Errorf("cannot parse workspace ID %q: %w", session.EditorSession.WorkspaceID, err)
Expand All @@ -69,8 +73,26 @@ func Run(ctx context.Context, o DbtInitOptions, d dependencies) (err error) {
return errors.Errorf("cannot fetch workspace credentials: %w", err)
}

// Build SandboxWorkspace from StorageWorkspace for env generation.
workspace := sandbox.WorkspaceFromStorage(storageWS, keboola.SandboxWorkspaceType(storageWS.StorageWorkspaceDetails.Backend))
// Build WorkspaceDetails combining both phases:
// Phase 1 fields (keboola_snowflake profile): BaseURL, BranchID, WorkspaceID
// Phase 2 fields (direct-Snowflake profile): Host, User, Database, Schema, Warehouse, PrivateKey
deref := func(s *string) string {
if s == nil {
return ""
}
return *s
}
Comment thread
Matovidlo marked this conversation as resolved.
Outdated
workspace := env.WorkspaceDetails{
Type: string(storageWS.StorageWorkspaceDetails.Backend),
Host: deref(storageWS.StorageWorkspaceDetails.Host),
User: deref(storageWS.StorageWorkspaceDetails.User),
Database: deref(storageWS.StorageWorkspaceDetails.Database),
Schema: deref(storageWS.StorageWorkspaceDetails.Schema),
Warehouse: deref(storageWS.StorageWorkspaceDetails.Warehouse),
BaseURL: o.BaseURL,
BranchID: branch.ID,
WorkspaceID: session.EditorSession.WorkspaceID,
}

// Determine private key from the freshly created credentials.
privateKey := ""
Expand Down
Loading