Skip to content

Commit 0232a9a

Browse files
andrewnesterandrewnester
authored
Do not exit early when checking incompatible tasks (#2692)
## Changes Do not exit early when checking incompatible tasks ## Why This allows us to find all tasks with local libraries and old DBR versions setup and not only the first one. ## Tests Existing tests pass <!-- If your PR needs to be included in the release notes for next release, add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->
1 parent d02b940 commit 0232a9a

5 files changed

Lines changed: 54 additions & 21 deletions

File tree

‎NEXT_CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@
99
### CLI
1010

1111
### Bundles
12+
* Do not exit early when checking incompatible tasks for specified DBR ([#2692](https://github.com/databricks/cli/pull/2692))
1213

1314
### API Changes

‎acceptance/bundle/trampoline/warning_message/output.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11

22
>>> errcode [CLI] bundle validate -t dev
3+
Error: task test_task uses cluster with incompatible DBR version 12.2.x-cpu-ml-scala2.12
4+
35
Error: Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.
46

57
Name: trampoline_warning_message
@@ -8,7 +10,7 @@ Workspace:
810
User: [USERNAME]
911
Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message/dev
1012

11-
Found 1 error
13+
Found 2 errors
1214

1315
Exit code: 1
1416

‎acceptance/bundle/trampoline/warning_message_with_old_spark/output.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11

22
>>> errcode [CLI] bundle validate
3+
Error: task test_task uses cluster with incompatible DBR version 7.3.x-scala2.12
4+
35
Error: Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.
46

57
Name: trampoline_warning_message_with_old_spark
@@ -8,6 +10,6 @@ Workspace:
810
User: [USERNAME]
911
Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message_with_old_spark/dev
1012

11-
Found 1 error
13+
Found 2 errors
1214

1315
Exit code: 1

‎bundle/trampoline/python_dbr_warning.go‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package trampoline
22

33
import (
44
"context"
5+
"fmt"
56
"strconv"
67
"strings"
78

@@ -29,22 +30,32 @@ func (m *wrapperWarning) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn
2930
return nil
3031
}
3132

32-
if hasIncompatibleWheelTasks(ctx, b) {
33-
return diag.Errorf("Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.")
33+
diags := hasIncompatibleWheelTasks(ctx, b)
34+
if len(diags) > 0 {
35+
diags = append(diags, diag.Diagnostic{
36+
Severity: diag.Error,
37+
Summary: "Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.",
38+
})
3439
}
35-
return nil
40+
return diags
3641
}
3742

3843
func isPythonWheelWrapperOn(b *bundle.Bundle) bool {
3944
return b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper
4045
}
4146

42-
func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool {
47+
func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
48+
var diags diag.Diagnostics
49+
4350
tasks := libraries.FindTasksWithLocalLibraries(b)
4451
for _, task := range tasks {
4552
if task.NewCluster != nil {
4653
if lowerThanExpectedVersion(task.NewCluster.SparkVersion) {
47-
return true
54+
diags = append(diags, diag.Diagnostic{
55+
Severity: diag.Error,
56+
Summary: fmt.Sprintf("task %s uses cluster with incompatible DBR version %s", task.TaskKey, task.NewCluster.SparkVersion),
57+
})
58+
continue
4859
}
4960
}
5061

@@ -53,7 +64,11 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool {
5364
for _, cluster := range job.JobClusters {
5465
if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster.SparkVersion != "" {
5566
if lowerThanExpectedVersion(cluster.NewCluster.SparkVersion) {
56-
return true
67+
diags = append(diags, diag.Diagnostic{
68+
Severity: diag.Error,
69+
Summary: fmt.Sprintf("job cluster %s uses incompatible DBR version %s", cluster.JobClusterKey, cluster.NewCluster.SparkVersion),
70+
})
71+
continue
5772
}
5873
}
5974
}
@@ -70,37 +85,41 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool {
7085
p, ok := dynvar.PureReferenceToPath(task.ExistingClusterId)
7186
if !ok || len(p) < 3 {
7287
log.Warnf(ctx, "unable to parse cluster key from %s", task.ExistingClusterId)
73-
return false
88+
continue
7489
}
7590

7691
if p[0].Key() != "resources" || p[1].Key() != "clusters" {
7792
log.Warnf(ctx, "incorrect variable reference for cluster id %s", task.ExistingClusterId)
78-
return false
93+
continue
7994
}
8095

8196
clusterKey := p[2].Key()
8297
cluster, ok := b.Config.Resources.Clusters[clusterKey]
8398
if !ok {
8499
log.Warnf(ctx, "unable to find cluster with key %s", clusterKey)
85-
return false
100+
continue
86101
}
87102
version = cluster.SparkVersion
88103
} else {
89104
version, err = getSparkVersionForCluster(ctx, b.WorkspaceClient(), task.ExistingClusterId)
90105
// If there's error getting spark version for cluster, do not mark it as incompatible
91106
if err != nil {
92107
log.Warnf(ctx, "unable to get spark version for cluster %s, err: %s", task.ExistingClusterId, err.Error())
93-
return false
108+
continue
94109
}
95110
}
96111

97112
if lowerThanExpectedVersion(version) {
98-
return true
113+
diags = append(diags, diag.Diagnostic{
114+
Severity: diag.Error,
115+
Summary: fmt.Sprintf("task %s uses cluster with incompatible DBR version %s", task.TaskKey, version),
116+
})
117+
continue
99118
}
100119
}
101120
}
102121

103-
return false
122+
return diags
104123
}
105124

106125
func lowerThanExpectedVersion(sparkVersion string) bool {

‎bundle/trampoline/python_dbr_warning_test.go‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ func TestIncompatibleWheelTasksWithNewCluster(t *testing.T) {
5050
},
5151
}
5252

53-
require.True(t, hasIncompatibleWheelTasks(context.Background(), b))
53+
diags := hasIncompatibleWheelTasks(context.Background(), b)
54+
require.NotEmpty(t, diags)
5455
}
5556

5657
func TestIncompatibleWheelTasksWithJobClusterKey(t *testing.T) {
@@ -99,10 +100,11 @@ func TestIncompatibleWheelTasksWithJobClusterKey(t *testing.T) {
99100
},
100101
}
101102

102-
require.True(t, hasIncompatibleWheelTasks(context.Background(), b))
103+
diags := hasIncompatibleWheelTasks(context.Background(), b)
104+
require.NotEmpty(t, diags)
103105

104-
diags := bundle.Apply(context.Background(), b, WrapperWarning())
105-
require.ErrorContains(t, diags.Error(), "require compute with DBR 13.3")
106+
diags = bundle.Apply(context.Background(), b, WrapperWarning())
107+
require.ErrorContains(t, diags.Error(), "uses incompatible DBR version 12.2.x-scala2.12")
106108
}
107109

108110
func TestIncompatibleWheelTasksWithExistingClusterId(t *testing.T) {
@@ -143,8 +145,13 @@ func TestIncompatibleWheelTasksWithExistingClusterId(t *testing.T) {
143145
clustersApi.EXPECT().GetByClusterId(mock.Anything, "test-key-1").Return(&compute.ClusterDetails{
144146
SparkVersion: "12.2.x-scala2.12",
145147
}, nil)
148+
clustersApi.EXPECT().GetByClusterId(mock.Anything, "test-key-2").Return(&compute.ClusterDetails{
149+
SparkVersion: "12.2.x-scala2.12",
150+
}, nil)
146151

147-
require.True(t, hasIncompatibleWheelTasks(context.Background(), b))
152+
diags := hasIncompatibleWheelTasks(context.Background(), b)
153+
require.NotEmpty(t, diags)
154+
require.ErrorContains(t, diags.Error(), "uses cluster with incompatible DBR version 12.2.x-scala2.12")
148155
}
149156

150157
func TestNoIncompatibleWheelTasks(t *testing.T) {
@@ -249,7 +256,8 @@ func TestNoIncompatibleWheelTasks(t *testing.T) {
249256
SparkVersion: "13.2.x-scala2.12",
250257
}, nil)
251258

252-
require.False(t, hasIncompatibleWheelTasks(context.Background(), b))
259+
diags := hasIncompatibleWheelTasks(context.Background(), b)
260+
require.Empty(t, diags)
253261
}
254262

255263
func TestTasksWithPyPiPackageAreCompatible(t *testing.T) {
@@ -289,7 +297,8 @@ func TestTasksWithPyPiPackageAreCompatible(t *testing.T) {
289297
m := mocks.NewMockWorkspaceClient(t)
290298
b.SetWorkpaceClient(m.WorkspaceClient)
291299

292-
require.False(t, hasIncompatibleWheelTasks(context.Background(), b))
300+
diags := hasIncompatibleWheelTasks(context.Background(), b)
301+
require.Empty(t, diags)
293302
}
294303

295304
func TestNoWarningWhenPythonWheelWrapperIsOn(t *testing.T) {

0 commit comments

Comments
 (0)