Skip to content

Commit c9569f2

Browse files
alyssa-dbalyssa-db
authored
Merge branch 'databricks:main' into cluster-id-config-1897
2 parents f366434 + d5108bf commit c9569f2

19 files changed

Lines changed: 273 additions & 58 deletions

File tree

‎acceptance/bundle/resources/pipelines/test.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1+
Timeout = '30s'
12
Cloud = true
23
Ignore = ["bar.py", "foo.py"]

‎acceptance/pipelines/install-pipelines-cli/output.txt‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
2+
=== install pipelines cli
3+
>>> errcode [CLI] install-pipelines-cli -d ./subdir
4+
pipelines successfully installed in directory "./subdir"
5+
6+
>>> errcode ./subdir/pipelines
7+
Pipelines CLI (stub, to be filled in)
8+
9+
Usage:
10+
pipelines [flags]
11+
12+
Flags:
13+
-h, --help help for pipelines
14+
15+
=== pipelines already installed
16+
>>> errcode [CLI] install-pipelines-cli -d ./subdir
17+
pipelines already installed in directory "./subdir"
18+
19+
=== pipelines file exists, should not overwrite
20+
>>> errcode [CLI] install-pipelines-cli -d ./subdir
21+
Error: cannot install pipelines CLI: "subdir/pipelines" already exists
22+
23+
Exit code: 1
24+
25+
=== databricks executable called with alias
26+
>>> errcode ./subdir/notdatabricks install-pipelines-cli -d ./subdir
27+
pipelines successfully installed in directory "./subdir"
28+
29+
>>> errcode ./subdir/pipelines
30+
Pipelines CLI (stub, to be filled in)
31+
32+
Usage:
33+
pipelines [flags]
34+
35+
Flags:
36+
-h, --help help for pipelines

‎acceptance/pipelines/install-pipelines-cli/script‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
tmpdir="./subdir"
2+
pipelines="$tmpdir/pipelines"
3+
mkdir -p $tmpdir
4+
5+
title "install pipelines cli"
6+
trace errcode $CLI install-pipelines-cli -d $tmpdir
7+
trace errcode $pipelines
8+
9+
title "pipelines already installed"
10+
trace errcode $CLI install-pipelines-cli -d $tmpdir
11+
rm -f $pipelines
12+
13+
title "pipelines file exists, should not overwrite"
14+
touch $pipelines
15+
trace errcode $CLI install-pipelines-cli -d $tmpdir
16+
rm -f $pipelines
17+
18+
title "databricks executable called with alias"
19+
cp $CLI $tmpdir/notdatabricks
20+
trace errcode $tmpdir/notdatabricks install-pipelines-cli -d $tmpdir
21+
trace errcode $pipelines
22+
23+
# Cleanup
24+
rm -f $tmpdir/notdatabricks $pipelines
25+
rm -rf $tmpdir

‎acceptance/pipelines/install-pipelines-cli/test.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Fix path with reverse slashes in the output for Windows.
2+
[[Repls]]
3+
Old = '\\\\'
4+
New = '/'
5+
6+
[[Repls]]
7+
Old = '\\'
8+
New = '/'

bundle/config/generate/app.go renamed to bundle/generate/app.go

File renamed without changes.

bundle/config/generate/dashboard.go renamed to bundle/generate/dashboard.go

File renamed without changes.

cmd/bundle/generate/utils.go renamed to bundle/generate/downloader.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,23 @@ import (
1818
"golang.org/x/sync/errgroup"
1919
)
2020

21-
type downloader struct {
21+
type Downloader struct {
2222
files map[string]string
2323
w *databricks.WorkspaceClient
2424
sourceDir string
2525
configDir string
2626
basePath string
2727
}
2828

29-
func (n *downloader) MarkTaskForDownload(ctx context.Context, task *jobs.Task) error {
29+
func (n *Downloader) MarkTaskForDownload(ctx context.Context, task *jobs.Task) error {
3030
if task.NotebookTask == nil {
3131
return nil
3232
}
3333

3434
return n.markNotebookForDownload(ctx, &task.NotebookTask.NotebookPath)
3535
}
3636

37-
func (n *downloader) MarkPipelineLibraryForDownload(ctx context.Context, lib *pipelines.PipelineLibrary) error {
37+
func (n *Downloader) MarkPipelineLibraryForDownload(ctx context.Context, lib *pipelines.PipelineLibrary) error {
3838
if lib.Notebook != nil {
3939
return n.markNotebookForDownload(ctx, &lib.Notebook.Path)
4040
}
@@ -46,7 +46,7 @@ func (n *downloader) MarkPipelineLibraryForDownload(ctx context.Context, lib *pi
4646
return nil
4747
}
4848

49-
func (n *downloader) markFileForDownload(ctx context.Context, filePath *string) error {
49+
func (n *Downloader) markFileForDownload(ctx context.Context, filePath *string) error {
5050
_, err := n.w.Workspace.GetStatusByPath(ctx, *filePath)
5151
if err != nil {
5252
return err
@@ -66,7 +66,7 @@ func (n *downloader) markFileForDownload(ctx context.Context, filePath *string)
6666
return nil
6767
}
6868

69-
func (n *downloader) markDirectoryForDownload(ctx context.Context, dirPath *string) error {
69+
func (n *Downloader) MarkDirectoryForDownload(ctx context.Context, dirPath *string) error {
7070
_, err := n.w.Workspace.GetStatusByPath(ctx, *dirPath)
7171
if err != nil {
7272
return err
@@ -102,7 +102,7 @@ func (n *downloader) markDirectoryForDownload(ctx context.Context, dirPath *stri
102102
return nil
103103
}
104104

105-
func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error {
105+
func (n *Downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error {
106106
info, err := n.w.Workspace.GetStatusByPath(ctx, *notebookPath)
107107
if err != nil {
108108
return err
@@ -123,7 +123,7 @@ func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *
123123
return nil
124124
}
125125

126-
func (n *downloader) relativePath(fullPath string) string {
126+
func (n *Downloader) relativePath(fullPath string) string {
127127
basePath := path.Dir(fullPath)
128128
if n.basePath != "" {
129129
basePath = n.basePath
@@ -138,7 +138,7 @@ func (n *downloader) relativePath(fullPath string) string {
138138
return relPath
139139
}
140140

141-
func (n *downloader) FlushToDisk(ctx context.Context, force bool) error {
141+
func (n *Downloader) FlushToDisk(ctx context.Context, force bool) error {
142142
// First check that all files can be written
143143
for targetPath := range n.files {
144144
info, err := os.Stat(targetPath)
@@ -185,8 +185,8 @@ func (n *downloader) FlushToDisk(ctx context.Context, force bool) error {
185185
return errs.Wait()
186186
}
187187

188-
func newDownloader(w *databricks.WorkspaceClient, sourceDir, configDir string) *downloader {
189-
return &downloader{
188+
func NewDownloader(w *databricks.WorkspaceClient, sourceDir, configDir string) *Downloader {
189+
return &Downloader{
190190
files: make(map[string]string),
191191
w: w,
192192
sourceDir: sourceDir,

‎bundle/generate/downloader_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package generate
2+
3+
import (
4+
"context"
5+
"path/filepath"
6+
"testing"
7+
8+
"github.com/databricks/databricks-sdk-go/experimental/mocks"
9+
"github.com/databricks/databricks-sdk-go/service/workspace"
10+
"github.com/stretchr/testify/assert"
11+
"github.com/stretchr/testify/require"
12+
)
13+
14+
func TestDownloader_MarkFileReturnsRelativePath(t *testing.T) {
15+
ctx := context.Background()
16+
m := mocks.NewMockWorkspaceClient(t)
17+
18+
dir := "base/dir/doesnt/matter"
19+
sourceDir := filepath.Join(dir, "source")
20+
configDir := filepath.Join(dir, "config")
21+
downloader := NewDownloader(m.WorkspaceClient, sourceDir, configDir)
22+
23+
var err error
24+
25+
// Test that the path is normalized to be relative to the config directory.
26+
f1 := "/a/b/c"
27+
m.GetMockWorkspaceAPI().EXPECT().GetStatusByPath(ctx, f1).Return(&workspace.ObjectInfo{
28+
Path: f1,
29+
}, nil)
30+
err = downloader.markFileForDownload(ctx, &f1)
31+
require.NoError(t, err)
32+
assert.Equal(t, filepath.FromSlash("../source/c"), f1)
33+
34+
// Test that the previous path doesn't influence the next path.
35+
f2 := "/a/b/c/d"
36+
m.GetMockWorkspaceAPI().EXPECT().GetStatusByPath(ctx, f2).Return(&workspace.ObjectInfo{
37+
Path: f2,
38+
}, nil)
39+
err = downloader.markFileForDownload(ctx, &f2)
40+
require.NoError(t, err)
41+
assert.Equal(t, filepath.FromSlash("../source/d"), f2)
42+
}

bundle/config/generate/job.go renamed to bundle/generate/job.go

File renamed without changes.

bundle/config/generate/pipeline.go renamed to bundle/generate/pipeline.go

File renamed without changes.

0 commit comments

Comments
 (0)