Skip to content

Commit 0106047

Browse files
radakamradakam
committed
Round-trip pipeline parameters and run_as in the test server
The fake pipelines handler unmarshals the create/update body into pipelines.PipelineSpec, which has no parameters or run_as fields (those live on CreatePipeline/EditPipeline and are echoed on the top-level GetPipelineResponse). So a re-read never returned them and the direct engine planned a perpetual update for any pipeline that set parameters/run_as. Decode those two request-only fields separately and set them on the stored GetPipelineResponse. Adds an acceptance test that deploys a pipeline with parameters and asserts the plan is a no-op.
1 parent c8c397f commit 0106047

7 files changed

Lines changed: 93 additions & 0 deletions

File tree

‎acceptance/bundle/resources/pipelines/drift/parameters/databricks.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
bundle:
2+
name: pipeline-parameters
3+
4+
resources:
5+
pipelines:
6+
my:
7+
name: test-pipeline
8+
parameters:
9+
my_param: my_value
10+
libraries:
11+
- file:
12+
path: "./foo.py"

‎acceptance/bundle/resources/pipelines/drift/parameters/foo.py‎

Whitespace-only changes.

‎acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎acceptance/bundle/resources/pipelines/drift/parameters/output.txt‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
2+
=== Initial deployment
3+
>>> [CLI] bundle deploy
4+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-parameters/default/files...
5+
Deploying resources...
6+
Updating deployment state...
7+
Deployment complete!
8+
9+
=== Plan is a no-op: pipeline parameters round-trip on read
10+
>>> [CLI] bundle plan
11+
Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
12+
13+
=== Redeploy is a no-op (no update call)
14+
>>> [CLI] bundle deploy
15+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-parameters/default/files...
16+
Deploying resources...
17+
Updating deployment state...
18+
Deployment complete!
19+
20+
>>> print_requests.py //api/2.0/pipelines
21+
json.method = "POST";
22+
json.path = "/api/2.0/pipelines";
23+
json.body.channel = "CURRENT";
24+
json.body.deployment.kind = "BUNDLE";
25+
json.body.deployment.metadata_file_path = "/Workspace/Users/[USERNAME]/.bundle/pipeline-parameters/default/state/metadata.json";
26+
json.body.edition = "ADVANCED";
27+
json.body.libraries[0].file.path = "/Workspace/Users/[USERNAME]/.bundle/pipeline-parameters/default/files/foo.py";
28+
json.body.name = "test-pipeline";
29+
json.body.parameters.my_param = "my_value";

‎acceptance/bundle/resources/pipelines/drift/parameters/script‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
echo "*" > .gitignore
2+
3+
title "Initial deployment"
4+
trace $CLI bundle deploy
5+
6+
title "Plan is a no-op: pipeline parameters round-trip on read"
7+
trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged"
8+
9+
title "Redeploy is a no-op (no update call)"
10+
trace $CLI bundle deploy
11+
trace print_requests.py //api/2.0/pipelines | gron.py | contains.py '!json.method = "PUT"'

‎acceptance/bundle/resources/pipelines/drift/parameters/test.toml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
RecordRequests = true
2+
3+
# Restrict to the direct engine: the round-trip gap this test guards lives in the
4+
# direct engine's read-back path (GetPipelineResponse.Parameters), and terraform
5+
# tracks parameters in its own state so it does not exercise the same path. Mirrors
6+
# catalogs/drift/managed_properties.
7+
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

‎libs/testserver/pipelines.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,22 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response {
5252
var r pipelines.GetPipelineResponse
5353
r.Spec = &spec
5454

55+
// parameters and run_as are request-only fields on CreatePipeline; they are not
56+
// part of PipelineSpec, so the unmarshal above silently drops them. The backend
57+
// echoes them back on the top-level GetPipelineResponse.Parameters/RunAs fields,
58+
// so decode them from the request separately and round-trip them here. Without
59+
// this a re-read never returns parameters/run_as, and the CLI plans a perpetual
60+
// update for any pipeline that sets them.
61+
var create pipelines.CreatePipeline
62+
if err := json.Unmarshal(req.Body, &create); err != nil {
63+
return Response{
64+
Body: fmt.Sprintf("cannot unmarshal request body: %s", err),
65+
StatusCode: 400,
66+
}
67+
}
68+
r.Parameters = create.Parameters
69+
r.RunAs = create.RunAs
70+
5571
pipelineId := nextUUID()
5672
r.PipelineId = pipelineId
5773
r.CreatorUserName = "tester@databricks.com"
@@ -100,7 +116,20 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response
100116
}
101117
}
102118

119+
// parameters and run_as live on EditPipeline, not PipelineSpec, so decode them
120+
// from the request separately and round-trip them on the top-level response
121+
// fields the CLI reads back (see PipelineCreate for the full rationale).
122+
var edit pipelines.EditPipeline
123+
if err := json.Unmarshal(req.Body, &edit); err != nil {
124+
return Response{
125+
Body: fmt.Sprintf("internal error: %s", err),
126+
StatusCode: 400,
127+
}
128+
}
129+
103130
item.Spec = &spec
131+
item.Parameters = edit.Parameters
132+
item.RunAs = edit.RunAs
104133
setSpecDefaults(&spec, pipelineId)
105134
s.Pipelines[pipelineId] = item
106135

0 commit comments

Comments
 (0)